{ "test_case_id": "A11Y-QC-004", "title": "Inter-rater reliability threshold", "category": "quality_control", "procedure_version": "2026.08.27-rc.9", "rule_type": "quality_control", "wcag_criteria": [], "level": "advisory", "risk": "medium", "risk_basis": "Checks accessibility programme quality control for inter-rater reliability threshold. People most likely affected: people affected by assessment decisions whose reliability depends on competent and consistent reviewer practice. Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. In the recorded accessibility-quality-control environment, use frozen fixtures, independent scoring and documented reconciliation; unresolved assessor disagreement can turn the same evidence into inconsistent client findings.", "review_status": "draft", "reviewed_at": "", "review_note": "Template content is subject to a hash-bound internal consistency review. Formal assessment claims require competent execution, current evidence and the applicable legal, participant and organisational authority.", "method_label": "Quality-control review", "tester_level": "Accessibility quality lead", "estimated_duration": "Typically 125–181 minutes for the stated single-target assumptions; use the component breakdown when estimating a real sample.", "execution_class": "manual", "automation_support": "none", "simulation_policy": "simulation_may_support_but_not_replace_manual_execution", "plain_summary": "Checks accessibility programme quality control for inter-rater reliability threshold.", "objective": "Verify accessibility programme quality control for inter-rater reliability threshold.", "test_subject": "The frozen procedure, evidence set, assigned roles and quality threshold.", "test_targets": [ "The quality-control record relevant to Inter-rater reliability threshold." ], "applicability": "Apply Inter-rater reliability threshold when the frozen procedure, evidence set, assigned roles and quality threshold. The selected focus is Inter-rater reliability threshold.", "applicability_questions": [ "Does the frozen scope contain a target relevant to Inter-rater reliability threshold?", "Are the required real environments, versions and settings identified separately from diagnostic simulations?", "Which default, revealed, error, interruption, recovery and confirmation states can change the result?", "Does a scope exclusion need to be reported separately from target applicability?" ], "inapplicability_conditions": [ "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required." ], "assumptions": [ "The selected scope identifies the product version, technologies, states and important user journeys being evaluated.", "The tester knows how to use this method and any required assistive technology.", "Known browser or assistive-technology limitations are recorded instead of being assigned to the product without evidence." ], "accessibility_support_assumptions": [ "The result covers only the browsers, platforms, readers and assistive technologies that were actually tested.", "Do not apply a result from one technology combination to an untested combination or to every user." ], "prerequisites": [ "The product area and test targets are agreed before testing starts.", "The operating system, browser, viewport and input method are recorded.", "Test data is safe to use and does not expose unnecessary personal information." ], "required_environment_fields": [ "catalogue_version", "review_protocol", "review_protocol_version", "calibration_set", "calibration_set_version", "evidence_set_reference", "reviewer_count" ], "steps": [ { "step_id": "S1", "title": "Prepare the scope, target and environment", "phase": "prepare", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold, the product version and scope are frozen; every selected target and material starting state is uniquely identified.", "tool_or_command": "Use the scope record and worksheet environment fields for Inter-rater reliability threshold: catalogue_version, review_protocol, review_protocol_version, calibration_set, calibration_set_version, evidence_set_reference, reviewer_count.", "action": "Before executing Inter-rater reliability threshold, freeze the product version and scope. Inventory each applicable target, select the required starting states, prepare safe test data and record every required environment value. Complete any named known-good calibration before attributing an observation to the product.", "expected": "The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "failure_observation": "During Inter-rater reliability threshold, the target, starting state, environment or scope record remains ambiguous for the scenario named Prepare the scope, target and environment. The preparation result cannot be demonstrated: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "evidence_to_record": "Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required.", "decision_effect": "For Inter-rater reliability threshold, an incomplete setup blocks the affected execution or leaves the omitted target untested; it never creates a pass or product failure.", "recovery_or_repeat": "For Inter-rater reliability threshold, correct the setup gap, preserve the original blocked record and start a new execution observation before proceeding.", "novice_guidance": "For Inter-rater reliability threshold, do not begin from an undocumented browser tab, document copy, device state, participant plan or evidence set. Record the exact item you will test first.", "limitations_and_claim_boundary": "This scenario records the action named Prepare the scope, target and environment for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision.", "scenario_purpose": "For Inter-rater reliability threshold, prepare the named scenario: Prepare the scope, target and environment. Target: The quality-control record relevant to Inter-rater reliability threshold. The preparation must establish this result: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Prepare the scope, target and environment.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Prepare the scope, target and environment only when the evidence (Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required.) supports: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "failed": "Record failed when the action is completed and this failure observation is reproduced: During Inter-rater reliability threshold, the target, starting state, environment or scope record remains ambiguous for the scenario named Prepare the scope, target and environment. The preparation result cannot be demonstrated: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." } }, { "step_id": "S2", "title": "Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution", "phase": "test", "action": "Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution.", "expected": "The control cannot be tuned after results are known and is reproducible.", "evidence_to_record": "Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold, begin in the recorded default state for the selected Inter-rater reliability threshold target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-quality-control, Use the immutable calibration set, independent score sheets, predeclared threshold and versioned calculation or reconciliation log named by the quality-control plan.", "decision_effect": "For Inter-rater reliability threshold, compare the recorded observation with this expected result: The control cannot be tuned after results are known and is reproducible. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat this action for every required variation of Inter-rater reliability threshold and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For Inter-rater reliability threshold: Keep assessors' initial decisions independent. Reconcile only after scores are frozen, and preserve both the disagreement and its resolution.", "scenario_purpose": "For Inter-rater reliability threshold, execute the named scenario: Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution. Target: The quality-control record relevant to Inter-rater reliability threshold. Compare the observation with this result: The control cannot be tuned after results are known and is reproducible.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution.", "failure_observation": "After the tester completes the Inter-rater reliability threshold scenario named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution, the recorded observation for the target is inconsistent with the required result: The control cannot be tuned after results are known and is reproducible.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution only when the evidence (Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.) supports: The control cannot be tuned after results are known and is reproducible.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the Inter-rater reliability threshold scenario named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution, the recorded observation for the target is inconsistent with the required result: The control cannot be tuned after results are known and is reproducible.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S3", "title": "Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes", "phase": "test", "action": "Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes; compare with the engagement threshold defined before testing.", "expected": "Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "evidence_to_record": "Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold, begin in the recorded default state for the selected Inter-rater reliability threshold target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-quality-control, Use the immutable calibration set, independent score sheets, predeclared threshold and versioned calculation or reconciliation log named by the quality-control plan.", "decision_effect": "For Inter-rater reliability threshold, compare the recorded observation with this expected result: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat this action for every required variation of Inter-rater reliability threshold and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For Inter-rater reliability threshold: Keep assessors' initial decisions independent. Reconcile only after scores are frozen, and preserve both the disagreement and its resolution.", "scenario_purpose": "For Inter-rater reliability threshold, execute the named scenario: Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes. Target: The quality-control record relevant to Inter-rater reliability threshold. Compare the observation with this result: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes.", "failure_observation": "After the tester completes the Inter-rater reliability threshold scenario named Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes, the recorded observation for the target is inconsistent with the required result: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes only when the evidence (Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.) supports: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the Inter-rater reliability threshold scenario named Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes, the recorded observation for the target is inconsistent with the required result: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Calculate raw agreement and an appropriate chance-corrected measure for duplicate categorical outcomes for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S4", "title": "Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records", "phase": "test", "action": "Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records.", "expected": "The quality evidence is complete, append-only and independently reviewable.", "evidence_to_record": "Inputs, method, output, disagreement classification and threshold comparison.", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold, begin in the recorded default state for the selected Inter-rater reliability threshold target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-quality-control, Use the immutable calibration set, independent score sheets, predeclared threshold and versioned calculation or reconciliation log named by the quality-control plan.", "decision_effect": "For Inter-rater reliability threshold, compare the recorded observation with this expected result: The quality evidence is complete, append-only and independently reviewable. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat this action for every required variation of Inter-rater reliability threshold and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For Inter-rater reliability threshold: Keep assessors' initial decisions independent. Reconcile only after scores are frozen, and preserve both the disagreement and its resolution.", "scenario_purpose": "For Inter-rater reliability threshold, execute the named scenario: Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records. Target: The quality-control record relevant to Inter-rater reliability threshold. Compare the observation with this result: The quality evidence is complete, append-only and independently reviewable.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records.", "failure_observation": "After the tester completes the Inter-rater reliability threshold scenario named Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records, the recorded observation for the target is inconsistent with the required result: The quality evidence is complete, append-only and independently reviewable.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records only when the evidence (Inputs, method, output, disagreement classification and threshold comparison.) supports: The quality evidence is complete, append-only and independently reviewable.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the Inter-rater reliability threshold scenario named Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records, the recorded observation for the target is inconsistent with the required result: The quality evidence is complete, append-only and independently reviewable.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Record original independent decisions, calculation inputs, result, disagreements and corrective actions without replacing the source records for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S5", "title": "Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested", "phase": "test", "action": "Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested.", "expected": "A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate.", "evidence_to_record": "Original decisions, evidence, rationale and timestamps preserved without overwriting.", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold, begin in the final actionable state before submission and the resulting success or confirmation state; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-quality-control, Use the immutable calibration set, independent score sheets, predeclared threshold and versioned calculation or reconciliation log named by the quality-control plan.", "decision_effect": "For Inter-rater reliability threshold, compare the recorded observation with this expected result: A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat this action for every required variation of Inter-rater reliability threshold and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For Inter-rater reliability threshold: Keep assessors' initial decisions independent. Reconcile only after scores are frozen, and preserve both the disagreement and its resolution.", "scenario_purpose": "For Inter-rater reliability threshold, execute the named scenario: Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested. Target: The quality-control record relevant to Inter-rater reliability threshold. Compare the observation with this result: A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested.", "failure_observation": "After the tester completes the Inter-rater reliability threshold scenario named Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested, the recorded observation for the target is inconsistent with the required result: A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested only when the evidence (Original decisions, evidence, rationale and timestamps preserved without overwriting.) supports: A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the Inter-rater reliability threshold scenario named Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested, the recorded observation for the target is inconsistent with the required result: A failed quality gate cannot be silently waived or reported as passed; organisational release authority remains separate.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Complete a distinct internal quality review or record a fail/cant-tell outcome and block unsupported external claims until remediation is retested for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S6", "title": "Repeat material variations and recovery", "phase": "test", "test_target": "The quality-control record relevant to Inter-rater reliability threshold.", "required_state": "For Inter-rater reliability threshold: Default, revealed, loading, empty, validation error, interruption, recovery and confirmation where meaningful. Every required environment, viewport, orientation, language and authenticated state recorded in the frozen scope.", "tool_or_command": "For Inter-rater reliability threshold, use the same recorded method, then change only the state, environment or recovery condition being compared.", "action": "Repeat the Inter-rater reliability threshold action for each inventoried variation that can change the outcome. Include relevant default, revealed, error, interruption, recovery and confirmation states and complete the selected process from start to finish where this test can affect task completion.", "expected": "Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "failure_observation": "After the tester completes the Inter-rater reliability threshold scenario named Repeat material variations and recovery, the recorded observation for the target is inconsistent with the required result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "evidence_to_record": "Variation identifier; changed state or environment; start and end points; actual observation; recovery result; intervention; evidence reference and target outcome.", "decision_effect": "For Inter-rater reliability threshold, a reproduced mismatch fails the affected target; an omitted required variation remains untested; conflicting evidence remains cannot tell.", "recovery_or_repeat": "Repeat this action for every required variation of Inter-rater reliability threshold and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For Inter-rater reliability threshold, change one material condition at a time where practical. Do not assume that a passing default state covers errors, overlays, orientation changes, interruptions or recovery.", "limitations_and_claim_boundary": "This scenario records the action named Repeat material variations and recovery for Inter-rater reliability threshold in accessibility-quality-control. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision.", "scenario_purpose": "For Inter-rater reliability threshold, execute the named scenario: Repeat material variations and recovery. Target: The quality-control record relevant to Inter-rater reliability threshold. Compare the observation with this result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "user_impact": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. This scenario isolates that risk in the action named Repeat material variations and recovery.", "decision_rules": { "passed": "Record passed for the Inter-rater reliability threshold scenario named Repeat material variations and recovery only when the evidence (Variation identifier; changed state or environment; start and end points; actual observation; recovery result; intervention; evidence reference and target outcome.) supports: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the Inter-rater reliability threshold scenario named Repeat material variations and recovery, the recorded observation for the target is inconsistent with the required result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "not_applicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." } } ], "expectations": [ { "expectation_id": "E1", "statement": "Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "outcome_if_unmet": "failed" }, { "expectation_id": "E2", "statement": "Every selected target and meaningful state is tested using the stated method.", "outcome_if_unmet": "untested" }, { "expectation_id": "E3", "statement": "Another qualified tester can understand and reproduce the decision from the recorded evidence.", "outcome_if_unmet": "cantTell" } ], "exceptions": [ "Use only an exception stated in the relevant standard, or an agreed out-of-scope decision. Record the reason and evidence.", "A tool limitation, lack of time or unavailable assistive technology is a limitation or incomplete result, not a pass." ], "outcome_definitions": { "passed": "The applicable test target meets every expectation in this rule or procedure.", "failed": "The applicable test target does not meet at least one expectation in this rule or procedure.", "inapplicable": "The test subject contains no target that matches this rule or procedure's applicability.", "cantTell": "The tester cannot fully determine applicability or whether every expectation is met.", "untested": "The test subject or target was not evaluated using this rule or procedure." }, "examples": [ { "example_id": "A11Y-QC-004-EX-PASSED", "outcome": "passed", "scenario": "Representative calibration example, not client evidence. Procedure: Inter-rater reliability threshold. Target: The quality-control record relevant to Inter-rater reliability threshold. Action: Before executing Inter-rater reliability threshold, freeze the product version and scope. Inventory each applicable target, select the required starting states, prepare safe test data and record every required environment value. Complete any named known-good calibration before attributing an observation to the product. Evidence: Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required. Observed result: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup." }, { "example_id": "A11Y-QC-004-EX-FAILED", "outcome": "failed", "scenario": "Representative calibration example, not client evidence. Procedure: Inter-rater reliability threshold. Target: The quality-control record relevant to Inter-rater reliability threshold. The tester completes the recorded action, but the captured evidence does not demonstrate: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup. The failed target and state remain separate from unexecuted targets." }, { "example_id": "A11Y-QC-004-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "Representative scope example, not client evidence. The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required. The completed inventory is retained as the basis for the inapplicable outcome." }, { "example_id": "A11Y-QC-004-EX-CANTTELL", "outcome": "cantTell", "scenario": "Representative unresolved example, not client evidence. The target is applicable, but conflicting evidence or unresolved responsibility prevents a defensible expectation decision after recorded comparison. The record identifies the conflicting evidence and the expertise or comparison needed to resolve it." }, { "example_id": "A11Y-QC-004-EX-UNTESTED", "outcome": "untested", "scenario": "Representative execution-gap example, not client evidence. An applicable target, required state, complete process or named real environment was not executed. The missing action, state or environment remains open and is not converted to a pass." } ], "sampling_rule": "Apply this control to the preselected assessment sample. Do not change the sample or threshold after results are known.", "complete_process_requirement": "If this check is part of a user journey, test the selected journey from start to finish, including relevant errors and recovery. Record any path that was not tested.", "pass_criteria": "Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "fail_criteria": "Fail if any applicable sampled instance does not satisfy: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "evidence_requirements": [ "Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.", "Original decisions, evidence, rationale and timestamps preserved without overwriting.", "Inputs, method, output, disagreement classification and threshold comparison.", "Independent agent-verification decision, verifier identity/version, scope, limitations, corrective action and next verification date." ], "official_source_ids": [ "W3C-WCAG-EM-1", "W3C-WCAG-EM-2", "W3C-ACT-RULES-FORMAT" ], "act_rule_mapping": { "format_alignment": "This is a project-authored manual procedure, not a W3C Approved ACT Rule.", "approved_rule_ids": [], "approved_rules": [], "mapping_status": "No supporting ACT rule selected", "registry_reviewed_at": "2026-08-27", "registry_source_id": "W3C-ACT-RULES", "mapping_rationale": "An ACT result can support only the part of this procedure covered by that rule. It does not replace the remaining manual or assistive-technology checks." }, "technique_traceability": [], "platform_ids": [ "accessibility-quality-control" ], "review_requirement": "A qualified accessibility reviewer must confirm the scope, evidence and result before it is reported as final.", "known_issues": [ "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "change_log": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Case-specific specialist content, evidence and platform boundaries completed for hash-bound agent verification." }, { "version": "2026.08.27-rc.9", "change": "Removed administrative filler steps, added contextual risk and result boundaries, and prepared the concise worksheet flow for internal consistency review." } ], "tags": [ "calibration", "distinct-quality-review", "quality-control", "quality_control" ], "status": "draft", "authoring_readiness": "content_complete", "short_title": "Inter-rater reliability threshold", "catalogue_version": "2026.08.27-rc.9", "identity": { "stable_case_id": "A11Y-QC-004", "procedure_title": "Inter-rater reliability threshold", "short_user_centred_title": "Inter-rater reliability threshold", "procedure_version": "2026.08.27-rc.9", "catalogue_version": "2026.08.27-rc.9", "status": "draft", "category": "quality_control", "procedure_type": "quality_control", "execution_class": "manual", "intended_tester_expertise": "Accessibility quality lead", "effort_model": { "model": "component_effort_v2", "small_scope_minutes": { "minimum": 125, "maximum": 181 }, "warning": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "calculation_guidance": "Add the recorded setup, per-target execution, evidence and technical-review components; then apply the stated variation rule.", "unit": "minutes", "setup": { "typical": 30, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 45, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 20, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 30, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 125, "maximum": 181 }, "assumptions": "one frozen review set and one reviewer pair; disagreements, adjudication and reruns are additional effort", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "related_procedures": [], "replaces": [], "replaced_by": [] }, "purpose_and_user_impact": { "barrier_detected": "Checks accessibility programme quality control for inter-rater reliability threshold.", "people_likely_affected": [ "People affected by assessment decisions whose reliability depends on competent and consistent reviewer practice." ], "functionality_or_information_at_risk": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable.", "result_scope": "quality_control", "claim_boundary": "This procedure determines whether Inter-rater reliability threshold meets its stated expectations for the recorded targets and accessibility-quality-control. Unexecuted targets, states and environments remain outside that result.", "why_it_matters": "Uncontrolled methods, conflicts or disagreement can make assessment conclusions unreliable. The procedure is necessary because checks accessibility programme quality control for inter-rater reliability threshold and the result can otherwise conceal a material barrier for the affected people named here." }, "requirement_mappings": [ { "source_id": "W3C-WCAG-EM-1", "requirement_id": "W3C-WCAG-EM-1", "requirement_level": "informative", "requirement_type": "advisory", "coverage_type": "supporting", "canonical_source": "https://www.w3.org/TR/WCAG-EM/", "source_status": "group_note", "source_version_or_publication_date": "WCAG-EM 1.0; W3C Note 10 July 2014", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] }, { "source_id": "W3C-WCAG-EM-2", "requirement_id": "W3C-WCAG-EM-2", "requirement_level": "informative", "requirement_type": "advisory", "coverage_type": "supporting", "canonical_source": "https://www.w3.org/TR/wcag-em-2/", "source_status": "draft_note", "source_version_or_publication_date": "WCAG-EM 2.0; Draft Note 5 February 2026", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] }, { "source_id": "W3C-ACT-RULES-FORMAT", "requirement_id": "W3C-ACT-RULES-FORMAT", "requirement_level": "informative", "requirement_type": "advisory", "coverage_type": "supporting", "canonical_source": "https://www.w3.org/TR/act-rules-format/", "source_status": "recommendation", "source_version_or_publication_date": "ACT Rules Format 1.1; W3C Recommendation 2026", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] } ], "applicability_model": { "test_subject": "The frozen procedure, evidence set, assigned roles and quality threshold.", "test_targets": [ "The quality-control record relevant to Inter-rater reliability threshold." ], "objective_questions": [ "Does the frozen scope contain a target relevant to Inter-rater reliability threshold?", "Are the required real environments, versions and settings identified separately from diagnostic simulations?", "Which default, revealed, error, interruption, recovery and confirmation states can change the result?", "Does a scope exclusion need to be reported separately from target applicability?" ], "subjective_judgements": [ "Record any judgement about meaning, equivalence, task success, platform responsibility or exception use with evidence for independent review." ], "inapplicability_conditions": [ "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required." ], "scope_exclusions": [ "An agreed exclusion is a reporting limitation, not an inapplicable target and not a pass." ], "required_states_and_variations": [ "Default, revealed, loading, empty, validation error, interruption, recovery and confirmation where meaningful.", "Every required environment, viewport, orientation, language and authenticated state recorded in the frozen scope." ], "complete_process_dependencies": "Run each selected critical process from its defined start through success, at least one representative error, interruption and recovery, and confirmation where the focus can affect task completion.", "conforming_alternate_version_considerations": "A conforming alternate version can affect a WCAG report only when all WCAG conformance conditions are verified separately. It does not erase a platform or usability observation." }, "normative_exceptions": [], "normative_exceptions_absent_reason": "This platform, journey, usability or quality procedure creates no additional normative exception. Apply exceptions only in the linked atomic or criterion-level conformance procedure and preserve that separate decision.", "environment": { "required_tester_expertise": "Accessibility quality lead", "required_platforms": [ "accessibility-quality-control" ], "minimum_environment_fields": [ "catalogue_version", "review_protocol", "review_protocol_version", "calibration_set", "calibration_set_version", "evidence_set_reference", "reviewer_count" ], "product_and_content_version_required": true, "viewport_and_orientation_required": true, "input_mode_required": true, "relevant_settings": [ "Catalogue and procedure version", "Frozen fixtures/sample", "Assigned roles", "Predeclared threshold", "Calculation version" ], "known_platform_limitations": [ "Quality-control evidence measures method reliability; it is not product accessibility or conformance evidence.", "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "clean_state_requirements": [ "Complete a known-good calibration in the same environment before product testing.", "Start from a recorded clean or known state and preserve any prior preferences, extensions, cache or profile settings that can affect output." ], "test_data_and_privacy_constraints": [ "Use authorised synthetic or controlled data.", "Do not record credentials, secrets or unnecessary personal data." ], "coverage_boundary": "Quality-control evidence measures method reliability; it is not product accessibility or conformance evidence." }, "decision_logic": { "target_passed": "Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "target_failed": "An applicable Inter-rater reliability threshold target has a reproducible material barrier against one or more stated expectations; identify the exact expectation, state and evidence.", "target_inapplicable": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "target_cant_tell": "The target is applicable, but conflicting evidence or unresolved responsibility prevents a defensible expectation decision after recorded comparison.", "target_untested": "An applicable target, required state, complete process or named real environment was not executed.", "aggregation": "Preserve every target/environment outcome. Any failed target prevents an overall passed procedure result. cantTell and untested remain unresolved; inapplicable is valid only when the completed inventory contains no applicable target.", "wcag_requirement_decision": "This specialist, platform, document, usability or quality result does not set a WCAG success criterion to satisfied or not_satisfied automatically. Use linked atomic and criterion-level procedures for that separate decision.", "conforming_alternate_version_effect": "Record alternate-version evidence separately. It does not erase a barrier in this tested version or environment.", "passed_automation_follow_up": "A checker, accessibility-tree inspection, emulator or simulated assistive technology is diagnostic evidence only. Complete every required manual, real-platform, contextual and exception check.", "platform_responsibility": "Compare the product with a known-good control and an authorised alternate environment. Record product defect, external limitation or unresolved responsibility only when evidence supports it; otherwise use cantTell.", "claim_readiness": "Schema validation or a passed target never creates a conformance or release claim. Required executions, a distinct competent review, limitations and any applicable legal, participant or organisational authority remain separate gates." }, "evidence_specification": [ { "evidence_id": "EV1", "method": "frozen_input", "record": "Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV2", "method": "independent_results", "record": "Original decisions, evidence, rationale and timestamps preserved without overwriting.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV3", "method": "calculation_record", "record": "Inputs, method, output, disagreement classification and threshold comparison.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV4", "method": "decision_record", "record": "Independent agent-verification decision, verifier identity/version, scope, limitations, corrective action and next verification date.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true } ], "calibration_examples": [ { "example_id": "A11Y-QC-004-EX-PASS", "outcome": "passed", "scenario": "In accessibility-quality-control, the selected Inter-rater reliability threshold target completes the recorded action and meets this expectation: Agreement meets the declared threshold or results are paused for recalibration, expanded review and documented limitation.", "explanation": "The example is a calibration scenario, not an execution or approval.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-QC-004-EX-FAIL", "outcome": "failed", "scenario": "An applicable Inter-rater reliability threshold target produces a reproducible barrier during the named action, with the exact state and affected user task recorded.", "explanation": "A failed target remains failed even if another environment or target passes.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-QC-004-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "The governed assessment has no quality-control activity relevant to Inter-rater reliability threshold; record why the control is not required.", "explanation": "Inapplicable requires a completed inventory showing that no target matches applicability; a missing environment is not inapplicable.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-QC-004-EX-CANTTELL", "outcome": "cantTell", "scenario": "The Inter-rater reliability threshold target is applicable, but conflicting product/platform evidence or missing authoritative information prevents a defensible responsibility or expectation decision.", "explanation": "Record the conflict, comparisons attempted and person or evidence needed to resolve it.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-QC-004-EX-UNTESTED", "outcome": "untested", "scenario": "The Inter-rater reliability threshold target is applicable, but a required state, complete process or real environment was not executed.", "explanation": "Untested remains incomplete and cannot be converted to passed or inapplicable.", "fixture_id": "", "fixture_version": "", "machine_testable": false } ], "review_record": { "author": "", "method_reviewer": "", "independent_standards_reviewer": "", "accessibility_ui_reviewer": "", "author_date": "", "method_review_date": "", "standards_review_date": "", "ui_review_date": "", "source_versions_reviewed": [], "exceptions_reviewed": false, "known_limitations": [ "Quality-control evidence measures method reliability; it is not product accessibility or conformance evidence.", "Real execution evidence and any customer-required legal, participant or organisational authority remain separate." ], "open_issues": [ "Execute all required real environments and calibration cases.", "Complete a distinct competent review and retain any unresolved material finding as blocked or cantTell." ], "decision": "pending", "next_review_date": "", "status_transition_reason": "", "release_gate_id": "", "release_gate_completed_at": "", "approval_authority": "", "approval_date": "" }, "privacy_and_security": { "browser_storage": "Worksheet data is stored only in this browser's local storage until the user exports or clears it.", "network_transmission": "The static library does not upload worksheet data.", "data_minimisation": "Record only evidence necessary to reproduce and review the decision.", "redaction": "Remove credentials, secrets and unnecessary personal or participant data before export or sharing.", "retention_and_deletion": "Apply the engagement retention schedule and clear local browser data when it is no longer required." }, "version_history": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Migrated to professional catalogue schema v3 with separated outcomes, structured authority, applicability, evidence, decision and review records." }, { "version": "2026.08.27-rc.9", "change": "Specialist procedure mapped to its task family and current first-party platform guidance and checked for internal consistency; competent review and real execution remain separate." }, { "version": "2026.08.27-rc.9", "change": "Added explicit preparation and variation/recovery steps plus a bounded representative worked example for paid-template use." }, { "version": "2026.08.27-rc.9", "change": "Procedure guidance and presentation simplified without changing draft governance status." } ], "migration": { "from_schema": "accessibility-test-catalog.v2", "from_procedure_version": "2026.08.04.1", "migrated_at": "2026-08-27", "content_review_state": "commercial_completeness_upgrade_agent_verified", "stable_identifier_preserved": true, "legacy_url_preserved": true, "delivery_model": "professional_delivery_v2", "agent_content_validation": { "status": "complete", "method": "case_specific_semantic_and_operational_checks_v1", "checked_fields": [ "target", "state", "tool", "action", "expected_observation", "failure_observation", "evidence", "outcome_rules", "claim_boundary", "compact_examples" ], "human_approval_effect": "none" } }, "test_type": "quality_control", "related_procedures": [], "platform_validation": { "required": false, "evidence_status": "not_executed", "real_device_or_at_claimed": false, "required_environment_ids": [ "accessibility-quality-control" ], "calibration_required": true, "evidence_refs": [], "limitations": "Quality-control evidence measures method reliability; it is not product accessibility or conformance evidence." }, "effort_guidance": { "unit": "minutes", "setup": { "typical": 30, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 45, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 20, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 30, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 125, "maximum": 181 }, "assumptions": "one frozen review set and one reviewer pair; disagreements, adjudication and reruns are additional effort", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "worked_examples": [ { "example_id": "A11Y-QC-004-WORKED-001", "example_kind": "representative_synthetic_example", "context": "Representative synthetic example for Inter-rater reliability threshold on a fictional, version-frozen product target; no client or participant execution occurred.", "inspected": "The tester follows S2 (Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution) using the target, required state, tool and repeat rule stated in this procedure.", "observed_output": "After the tester completes the Inter-rater reliability threshold scenario named Freeze versions, sample, metric, threshold, agent identities and conflicts before quality-control execution, the recorded observation for the target is inconsistent with the required result: The control cannot be tuned after results are known and is reproducible.", "evidence_captured": [ "example-evidence-001: target and state record for The quality-control record relevant to Inter-rater reliability threshold.", "example-evidence-002: contemporaneous observation from the named method — Procedure, catalogue, fixture/sample, assigned roles, conflicts, metric and threshold versions.", "example-evidence-003: comparison note linking the actual observation to the expected result and claim boundary" ], "tester_reasoning": "The recorded observation does not demonstrate the expected result: The control cannot be tuned after results are known and is reproducible. The failed decision is limited to this synthetic target and state; unexecuted coverage is not inferred.", "decision": "failed", "completed_finding": { "finding_id": "A11Y-QC-004-example-finding-001", "title": "Inter-rater reliability threshold: expected result is not demonstrated", "severity": "medium", "user_impact": "The represented barrier may affect People affected by assessment decisions whose reliability depends on competent and consistent reviewer practice.", "evidence_refs": [ "example-evidence-001", "example-evidence-002", "example-evidence-003" ] }, "remediation_advice": "Change the represented target or process so it demonstrates this result: The control cannot be tuned after results are known and is reproducible. Then rerun the affected step and every dependent variation.", "reviewer_feedback": "The reviewer confirms applicability, environment, actual-versus-expected comparison, evidence references and bounded wording; unsupported product-wide or conformance language is removed.", "final_client_report_wording": "Synthetic training finding: the sampled Inter-rater reliability threshold target does not meet the procedure expectation in the recorded state. User impact and broader scope require evidence from the buyer's real execution.", "retest_outcome": "Synthetic retest example: after the represented change, repeat the failed step plus material error and recovery states. Close the finding only when new evidence demonstrates the expected result; otherwise retain the non-pass outcome.", "claim_boundary": "This is a representative, synthetic worked example for training and calibration. It is not evidence from a client product, participant or real assistive-technology execution." } ] }