{
  "metadata": {
    "version": "0.1.0",
    "created_on": "2026-09-28",
    "title": "Rubric Diversity Atlas",
    "description": "Initial diverse task/rubric specimens and correct-response variation probes.",
    "counts": {
      "total": 30,
      "by_family": {
        "communication_instruction_following": 6,
        "database_business_operations": 3,
        "mathematical_statistical_analysis": 6,
        "planning_scheduling": 2,
        "research_document_synthesis": 1,
        "retrieval_data_curation": 1,
        "scientific_investigation": 3,
        "software_repair": 5,
        "terminal_data_pipelines": 2,
        "user_preferences": 1
      },
      "by_source": {
        "AI-MO/NuminaMath-1.5": 2,
        "JunkaiZ/Rubrics": 2,
        "ScaleAI/researchrubrics": 8,
        "XiaomiMiMo/MiMo-V2.6-RL-oss": 4,
        "agentscope-ai/Auto-Rubric": 2,
        "google-research-datasets/mbpp": 3,
        "gretelai/synthetic_text_to_sql": 2,
        "nebius/SWE-rebench-V2": 2,
        "nvidia/Nemotron-RL-Science-v1": 2,
        "openai/gsm8k": 3
      },
      "by_pool": {
        "proposed": 18,
        "current": 12
      },
      "by_origin": {
        "drafted": 14,
        "source-provided": 16
      },
      "by_language": {
        "en": 27,
        "zh": 3
      },
      "represented_leaves": 17,
      "response_probes": 93,
      "full_response_probes": 52,
      "examples_with_full_responses": 18,
      "examples_with_executed_checks": 9
    },
    "review_limits": [
      "Purposive pilot selected to expose coverage and rubric failure modes. Counts are not a representative estimate of source quality or task diversity.",
      "Task-family assignments are provisional editorial annotations. Family coverage does not imply coverage of all 40 planned leaves or of task difficulty.",
      "Source rubrics are preserved; proposed revisions and solution-derived rubrics are explicitly marked as drafts. None of these specimens has been accepted into training.",
      "Full responses, response sketches, editorial expectations and executed checks are different evidence levels. No model rubric-judge acceptance rates were measured.",
      "Execution verifies only the stated tests or arithmetic. It does not prove arbitrary natural-language reasoning or acceptance of every valid response.",
      "GSM8K and MBPP are established evaluation benchmarks. These train-split examples are demonstration material; decontaminate against your evaluation policy before any future training use.",
      "MiMo examples require source environments for outcome verification. No missing workspace was reconstructed; no environment-dependent response is certified correct.",
      "DeepResearch Bench II is omitted from this public sample pending verification of its upstream provenance. Current and proposed datasets can share upstream tasks; counts here are distinct specimens, not proven distinct lineages."
    ],
    "inventory_sha256_unchanged": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5"
  },
  "taxonomy": {
    "schema_version": "1.0",
    "taxonomy_version": "1.0",
    "created_on": "2026-09-28",
    "status": "proposed_planning_artifact",
    "runtime_integration": "Not consumed by the collection CLI. Existing family identifiers and corpus requirements remain authoritative.",
    "purpose": "Plan a diverse candidate mixture and auditable quality evaluation without importing, relabeling or accepting tasks.",
    "pillars": [
      {
        "id": "build_operate",
        "label": "Build and operate systems",
        "family_ids": [
          "software_repair",
          "terminal_data_pipelines",
          "database_business_operations"
        ]
      },
      {
        "id": "evidence_investigation",
        "label": "Establish and synthesize evidence",
        "family_ids": [
          "retrieval_data_curation",
          "research_document_synthesis",
          "scientific_investigation"
        ]
      },
      {
        "id": "analysis_coordination",
        "label": "Analyze quantities and coordinate actions",
        "family_ids": [
          "mathematical_statistical_analysis",
          "planning_scheduling"
        ]
      },
      {
        "id": "communication_adaptation",
        "label": "Communicate and adapt to users",
        "family_ids": [
          "communication_instruction_following",
          "user_preferences"
        ]
      }
    ],
    "families": [
      {
        "id": "software_repair",
        "pillar": "build_operate",
        "label": "Software repair",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "software_repair.functional_bug_repair",
            "label": "Functional bug repair",
            "definition": "Correct incorrect program behavior in an existing codebase.",
            "illustrative_example": "Repair an error-propagation bug and preserve existing behavior.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "software_repair.performance_resource_repair",
            "label": "Performance and resource repair",
            "definition": "Restore required latency, memory, throughput or resource behavior.",
            "illustrative_example": "Fix a leak or avoid repeated expensive work without changing results.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "software_repair.interface_compatibility_repair",
            "label": "Interface and compatibility repair",
            "definition": "Restore an API, protocol, serialization or platform contract.",
            "illustrative_example": "Restore compatibility with a changed dependency API.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "software_repair.build_dependency_repair",
            "label": "Build and dependency repair",
            "definition": "Repair a repository's build, dependency configuration or development infrastructure.",
            "illustrative_example": "Repair a dependency conflict that prevents the project's tests from running.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "terminal_data_pipelines",
        "pillar": "build_operate",
        "label": "Terminal and data pipelines",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "terminal_data_pipelines.environment_service_setup",
            "label": "Environment and service setup",
            "definition": "Configure an operational environment or service with observable readiness.",
            "illustrative_example": "Configure a local service and demonstrate its health checks.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "terminal_data_pipelines.file_artifact_processing",
            "label": "File and artifact processing",
            "definition": "Transform filesystem artifacts with specified operational behavior.",
            "illustrative_example": "Convert and consolidate files into validated output artifacts.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "terminal_data_pipelines.pipeline_automation",
            "label": "Pipeline automation",
            "definition": "Build a repeatable executable workflow whose end-to-end behavior is evaluated.",
            "illustrative_example": "Produce an idempotent ingestion and reporting pipeline.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "terminal_data_pipelines.runtime_diagnosis_recovery",
            "label": "Runtime diagnosis and recovery",
            "definition": "Diagnose and restore execution of an environment, service or running workflow.",
            "illustrative_example": "Find why a scheduled pipeline fails and restore its execution.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "database_business_operations",
        "pillar": "build_operate",
        "label": "Database and business operations",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "database_business_operations.database_query_analysis",
            "label": "Database querying and analysis",
            "definition": "Produce a correct database query, query result or database analysis.",
            "illustrative_example": "Join operational tables and calculate the requested report.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "database_business_operations.record_update_reconciliation",
            "label": "Record updates and reconciliation",
            "definition": "Create, update or reconcile authoritative operational records.",
            "illustrative_example": "Reconcile conflicting customer records and apply justified updates.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "database_business_operations.schema_integrity_management",
            "label": "Schema and integrity management",
            "definition": "Change schemas or enforce database constraints while preserving required data.",
            "illustrative_example": "Migrate a schema and demonstrate preservation of integrity constraints.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "database_business_operations.business_workflow_execution",
            "label": "Business workflow execution",
            "definition": "Complete a business procedure across record systems with correct state changes.",
            "illustrative_example": "Apply an approval procedure across a document registry and CRM.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "retrieval_data_curation",
        "pillar": "evidence_investigation",
        "label": "Retrieval and data curation",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "retrieval_data_curation.evidence_retrieval_ranking",
            "label": "Evidence retrieval and ranking",
            "definition": "Locate and prioritize material for a stated evidence need.",
            "illustrative_example": "Find the authoritative versions of documents for a specified question.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "retrieval_data_curation.structured_fact_extraction",
            "label": "Structured fact extraction",
            "definition": "Extract attributable entities, attributes or relations from supplied evidence.",
            "illustrative_example": "Extract a source-linked table of claims, dates and entities.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "retrieval_data_curation.data_cleaning_deduplication",
            "label": "Data cleaning and deduplication",
            "definition": "Validate, normalize and select data with explicit inclusion rules.",
            "illustrative_example": "Remove duplicates and ineligible records while retaining provenance.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "retrieval_data_curation.entity_record_resolution",
            "label": "Entity and record resolution",
            "definition": "Match identities across records and reconcile entity aliases.",
            "illustrative_example": "Resolve counsel and company aliases without inventing affiliations.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "research_document_synthesis",
        "pillar": "evidence_investigation",
        "label": "Research and document synthesis",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "research_document_synthesis.grounded_explanation_synthesis",
            "label": "Grounded explanation and synthesis",
            "definition": "Combine external evidence into a supported explanation or document.",
            "illustrative_example": "Write an evidence-grounded briefing from supplied documents.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "research_document_synthesis.comparative_evidence_review",
            "label": "Comparative evidence review",
            "definition": "Compare alternatives or findings across multiple sources.",
            "illustrative_example": "Compare competing approaches using consistent evidence-backed criteria.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "research_document_synthesis.evidence_based_recommendation",
            "label": "Evidence-based recommendation",
            "definition": "Recommend action based on external evidence and stated objectives.",
            "illustrative_example": "Recommend a policy option with supported tradeoffs and limitations.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "research_document_synthesis.claim_verification_reconciliation",
            "label": "Claim verification and reconciliation",
            "definition": "Assess claims and reconcile conflicting or versioned evidence.",
            "illustrative_example": "Resolve contradictory reported figures using authoritative source versions.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "scientific_investigation",
        "pillar": "evidence_investigation",
        "label": "Scientific investigation",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "scientific_investigation.mechanistic_explanation_prediction",
            "label": "Mechanistic explanation and prediction",
            "definition": "Use scientific mechanisms to explain or predict a phenomenon.",
            "illustrative_example": "Explain the observed behavior of a physical system from its mechanisms.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "scientific_investigation.experimental_design",
            "label": "Experimental design",
            "definition": "Specify an informative experiment or measurement with controls.",
            "illustrative_example": "Design a discriminating experiment and identify required controls.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "scientific_investigation.empirical_result_interpretation",
            "label": "Empirical result interpretation",
            "definition": "Interpret observations using domain theory and uncertainty.",
            "illustrative_example": "Interpret experimental observations and identify justified conclusions.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "scientific_investigation.hypothesis_causal_assessment",
            "label": "Hypothesis and causal assessment",
            "definition": "Distinguish hypotheses or assess causal explanations.",
            "illustrative_example": "Compare competing hypotheses against observed evidence and confounders.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "mathematical_statistical_analysis",
        "pillar": "analysis_coordination",
        "label": "Mathematical and statistical analysis",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "mathematical_statistical_analysis.numerical_derivation",
            "label": "Numerical derivation",
            "definition": "Derive correct quantities under mathematical or accounting rules.",
            "illustrative_example": "Reconcile tax amounts using stated rates and rounding rules.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "mathematical_statistical_analysis.proof_symbolic_reasoning",
            "label": "Proof and symbolic reasoning",
            "definition": "Prove a claim or perform justified symbolic reasoning.",
            "illustrative_example": "Prove an identity or derive a symbolic result.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "mathematical_statistical_analysis.statistical_estimation_inference",
            "label": "Statistical estimation and inference",
            "definition": "Estimate quantities and characterize uncertainty from data.",
            "illustrative_example": "Estimate an effect with justified assumptions and uncertainty.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "mathematical_statistical_analysis.optimization_numerical_modeling",
            "label": "Optimization and numerical modeling",
            "definition": "Formulate or analyze an optimization problem or numerical model.",
            "illustrative_example": "Formulate and solve a constrained numerical optimization problem.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "planning_scheduling",
        "pillar": "analysis_coordination",
        "label": "Planning and scheduling",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "planning_scheduling.schedule_resource_allocation",
            "label": "Scheduling and resource allocation",
            "definition": "Assign times and resources to produce a feasible plan.",
            "illustrative_example": "Produce an itinerary or rota satisfying time, availability and resource constraints.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "planning_scheduling.dependency_workflow_planning",
            "label": "Dependency and workflow planning",
            "definition": "Order actions and prerequisites in an executable real-world plan.",
            "illustrative_example": "Plan a migration with explicit dependencies and readiness criteria.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "planning_scheduling.contingency_risk_planning",
            "label": "Contingency and risk planning",
            "definition": "Prepare conditional responses and reserves for specified uncertainties.",
            "illustrative_example": "Develop contingency actions for likely disruptions and resource shortfalls.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "planning_scheduling.plan_monitoring_replanning",
            "label": "Plan monitoring and replanning",
            "definition": "Revise a plan after observing changed conditions.",
            "illustrative_example": "Update a schedule after a delay while preserving hard constraints.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "communication_instruction_following",
        "pillar": "communication_adaptation",
        "label": "Communication and instruction following",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "communication_instruction_following.content_composition",
            "label": "Content composition",
            "definition": "Compose communication or creative content for an explicit purpose.",
            "illustrative_example": "Draft a clear workplace message or creative piece for a stated audience.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "communication_instruction_following.editing_style_revision",
            "label": "Editing and style revision",
            "definition": "Revise supplied content for meaning, organization, purpose or audience.",
            "illustrative_example": "Revise an existing message using concrete feedback.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "communication_instruction_following.faithful_language_transformation",
            "label": "Faithful language transformation",
            "definition": "Translate or transform language while preserving relevant meaning.",
            "illustrative_example": "Translate or simplify supplied text without changing material claims.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "communication_instruction_following.format_constraint_generation",
            "label": "Format and constraint generation",
            "definition": "Generate content satisfying explicit structural or form requirements.",
            "illustrative_example": "Produce an output satisfying a specified schema and content constraints.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      },
      {
        "id": "user_preferences",
        "pillar": "communication_adaptation",
        "label": "User preferences",
        "accepted_target_requirement": 10000,
        "implied_minimum_underlying_tasks": 2000,
        "leaves": [
          {
            "id": "user_preferences.preference_elicitation",
            "label": "Preference elicitation",
            "definition": "Obtain missing user-specific tastes or priorities needed for a decision.",
            "illustrative_example": "Ask targeted questions that distinguish useful recommendation alternatives.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "user_preferences.personalized_selection_ranking",
            "label": "Personalized selection and ranking",
            "definition": "Select or rank alternatives using an identifiable user's preferences.",
            "illustrative_example": "Rank options against a supplied user profile and explain the fit.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "user_preferences.preference_tradeoff_resolution",
            "label": "Preference tradeoff resolution",
            "definition": "Resolve tensions among user-specific priorities with explicit evidence.",
            "illustrative_example": "Choose among competing options using the user's stated priority tradeoffs.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          },
          {
            "id": "user_preferences.preference_conditioned_adaptation",
            "label": "Preference-conditioned adaptation",
            "definition": "Adapt content or behavior using established user-specific preferences.",
            "illustrative_example": "Revise a recommendation after the user changes a stated preference.",
            "initial_accepted_target_aspiration": 2500,
            "minimum_tasks_at_perfect_five_variant_yield": 500
          }
        ]
      }
    ],
    "counting_policy": {
      "primary_assignment": "One validated primary family and leaf per underlying task. Secondary skills contribute descriptive coverage only.",
      "underlying_identity": "Use upstream task identity plus prompt, initial state and asset provenance. Identical text with different task states is not automatically a duplicate.",
      "expanded_records": "Responses, preference pairs, language translations, paraphrases and support variants do not create new underlying tasks.",
      "related_grouping": "Keep actual task/template/world/user relationships in group_keys. Coarse family, publisher and whole-lineage identifiers are concentration attributes, not automatic split groups.",
      "statuses_reported_separately": [
        "raw_candidates",
        "classified_candidates",
        "eligible_tasks",
        "accepted_underlying_tasks",
        "accepted_targets",
        "evaluation_ready_inputs",
        "accepted_seeds"
      ],
      "unknowns": "Unassigned or provisional classifications do not establish validated leaf coverage. Unknown lineage is reported as unknown, never assumed independent."
    },
    "assignment_rules": [
      "Classify the essential deliverable and dominant acceptance condition, independently of source name, domain, output prose or interface.",
      "Existing source-code repair maps to software_repair; operational environments and executable workflows map to terminal_data_pipelines; queries, schema, records and enterprise state changes map to database_business_operations.",
      "Terminal execution is an interface, not a family. A SQL query executed in a terminal remains database work.",
      "Repeatable workflow behavior maps to terminal_data_pipelines; the relevance, quality, identity and provenance of a data collection map to retrieval_data_curation.",
      "Locating or extracting attributable material maps to retrieval_data_curation; integrated explanations, comparisons and source-based recommendations map to research_document_synthesis.",
      "Scientific mechanisms, experiments and interpretation map to scientific_investigation; mathematical derivation, proof and estimation map to mathematical_statistical_analysis.",
      "A feasible real-world sequence or allocation maps to planning_scheduling; mathematical optimization formulation and analysis map to mathematical_statistical_analysis.",
      "User-specific preferences must materially affect the correct behavior for user_preferences. Human rankings of generic responses alone do not qualify.",
      "Communication is primary only when composition, revision, faithful language transformation or form compliance is the principal work.",
      "Within a family, select the most specific evidenced acceptance condition: compatibility before generic functional repair, explicit conflicting claims before general synthesis, changing conditions before static planning.",
      "If the primary outcome is genuinely unresolved, retain a provisional or unassigned classification and report the ambiguity; do not force a category to fill a quota.",
      "Arbitrary new-application or general code-generation tasks are outside the current software_repair scope unless another existing family accurately describes their primary outcome."
    ],
    "allocation": {
      "existing_requirements": {
        "accepted_targets_minimum": 100000,
        "accepted_underlying_tasks_minimum": 20000,
        "accepted_targets_per_family_minimum": 10000,
        "variants_per_task_maximum": 5,
        "max_additional_seed_examples": 10000,
        "final_test_counts_toward_accepted_target": false
      },
      "initial_plan": {
        "family_target_share": 0.1,
        "leaf_target_share_within_family": 0.25,
        "leaf_targets_are_aspirational": true,
        "leaf_reallocation": "Reallocate within families after the pilot using passing supply and measured quality. Publish every underfilled or uncovered leaf; do not lower quality gates.",
        "task_yield_caveat": "500 tasks can produce 2,500 accepted targets only if all five variants are accepted. Actual required task supply depends on measured accepted variants per task."
      },
      "proposed_breadth_alerts": {
        "max_leaf_share_of_family_targets": 0.5,
        "min_materially_represented_leaves_per_family": 3,
        "material_representation_minimum_family_target_share": 0.1,
        "enforcement": "Planning alerts, not implemented acceptance gates. Track all four leaves; an alert-free family may still have an explicitly reported fourth-leaf gap."
      },
      "proposed_concentration_caps": {
        "lineage_maximum_share": 0.25,
        "denominators": [
          "distinct_accepted_underlying_tasks",
          "accepted_targets",
          "serialized_training_tokens"
        ],
        "apply_to": "Each known upstream lineage including derivatives; count multi-origin records against every contributing lineage when auditing caps.",
        "publisher": "Report publisher concentration separately; NVIDIA is a publisher, not automatically one lineage.",
        "unknown_lineage": "Report its share separately and do not claim a lineage-balanced corpus while unresolved.",
        "enforcement": "Proposed sampler policy; not implemented by existing corpus validation."
      },
      "existing_seed_cap": {
        "rubrichub_maximum_fraction": 0.25,
        "denominators": [
          "all_accepted_seed_tokens",
          "training_seed_tokens"
        ],
        "note": "Existing rule, separate from the proposed general lineage caps. Future lineage-aware auditing must include derivatives."
      },
      "difficulty": "Keep unmeasured until solver outcomes exist. Allocate within-family difficulty bands after a development pilot; avoid a mixture consisting only of universally solved or universally failed tasks.",
      "required_reports": [
        "family_and_leaf_target_share",
        "family_and_leaf_underlying_task_share",
        "training_token_share",
        "source_and_lineage_concentration",
        "template_and_world_concentration",
        "domain_language_interface_and_reference_coverage",
        "quality_failure_reasons_and_unknowns",
        "difficulty_measurement_and_sample_size",
        "current_vs_proposed_source_contribution"
      ]
    },
    "facets": {
      "domains": "Open vocabulary such as finance, law, medicine, physics and logistics; domain does not determine family.",
      "languages": "BCP-47 language tags, including multiple languages when actually required; source labels need validation.",
      "tools": "Actual interaction tools such as filesystem, SQL, browser or business APIs; tool-call text alone does not establish tool execution.",
      "output_types": "Text, structured record, code patch, file artifact, database state, action plan or other observed deliverable.",
      "interaction_horizon": "Single response, bounded multi-step interaction or extended stateful workflow; classify from actual task requirements.",
      "reference_modes": [
        "executable",
        "rubric",
        "artifact",
        "combined"
      ],
      "evidence_origins": [
        "source_tests",
        "source_artifact",
        "source_rubric",
        "human_annotation",
        "model_generated_reference",
        "generated_rubric"
      ],
      "difficulty": "Measured separately from task quality, with solver/checkpoint, protocol, attempt count and outcome evidence."
    },
    "quality_gates": [
      {
        "id": "provenance_eligibility",
        "stage": "candidate",
        "required": "Training eligibility, original provenance, immutable source revision or file hash, and component terms are established."
      },
      {
        "id": "task_completeness",
        "stage": "candidate",
        "required": "Instructions are coherent and all required context, attachments, permissions and time assumptions are available."
      },
      {
        "id": "classification_identity",
        "stage": "candidate",
        "required": "Primary outcome is supported by task evidence; task identity and duplicate/lineage uncertainty are recorded."
      },
      {
        "id": "reference_scoring_fidelity",
        "stage": "candidate",
        "required": "Reference kind and original scoring are faithfully represented, including weights, penalties, gates and category aggregation."
      },
      {
        "id": "environment_readiness",
        "stage": "pilot",
        "required": "Required environments can be reproduced from pinned accessible assets; original verifiers are run through appropriate isolated runners."
      },
      {
        "id": "reference_behavior",
        "stage": "pilot",
        "required": "Known successful and controlled failing attempts receive justified outcomes; legitimate alternative solutions are not rejected by arbitrary formatting or method restrictions."
      },
      {
        "id": "rubric_grounding",
        "stage": "pilot",
        "required": "Criteria are supported by task-visible evidence and legitimate task requirements; unsupported additions, omissions and duplicate criteria are addressed."
      },
      {
        "id": "split_integrity",
        "stage": "collection",
        "required": "Task/template/world/user groups and known source derivatives stay within split boundaries; protected evaluation overlap is excluded."
      },
      {
        "id": "episode_acceptance",
        "stage": "collection",
        "required": "All existing support/selection separation, grounding, scoring, independent repeat, perturbation and corpus-integrity checks pass."
      }
    ],
    "quality_policy": {
      "statuses": [
        "unassessed",
        "blocked",
        "candidate_checks_passed"
      ],
      "gate_results": [
        "pass",
        "fail",
        "unknown",
        "not_applicable"
      ],
      "evidence_required": "Each pass or justified not_applicable result needs evidence; unknown never means pass. Keep a separate record of each quality dimension.",
      "non_compensation": "Coverage, source reputation, larger row counts, human labels or the presence of executable tests cannot compensate for a failed applicable quality gate.",
      "acceptance": "Candidate checks passing is not episode acceptance. Use the existing pipeline's accepted/evaluation_ready/quarantined states for generated episodes.",
      "privacy": "Task metadata, hidden references, tests and solution artifacts stay outside task-solving and rubric-writer training inputs unless explicitly part of the visible task."
    },
    "pilot": {
      "status": "planned_not_run",
      "candidate_count_aspiration": 1000,
      "candidates_per_leaf_aspiration": 25,
      "sampling": "Sample distinct task groups, stratified across available source lineages, domains and interfaces. Include current inventory where relevant and proposed sources where permitted in a future curation phase.",
      "scarcity": "Record unfilled leaf slots; do not substitute unrelated tasks. Any reallocation is reported explicitly.",
      "stages": [
        "Static task/reference/asset audit and primary classification",
        "Development-only environment and reference validation",
        "Bounded live attempt collection under a separately configured pilot",
        "Estimate acceptance, accepted variants per task, grading disagreement, difficulty and cost; revise the mixture"
      ],
      "uncertainty": "Twenty-five candidates per leaf is a diagnostic sample, not a precise acceptance-rate estimate. Report sample sizes and uncertainty; use sampling weights for aggregate estimates.",
      "no_new_human_annotations": "Use source evidence and automated validation; unresolved cases remain blocked. Human annotation is not a required step.",
      "evaluation": "Keep final evaluation groups frozen and outside pilot-driven tuning."
    }
  },
  "examples": [
    {
      "id": "verified:mbpp:repair:605",
      "title": "Repair a reference implementation that misclassifies primes",
      "source": {
        "dataset": "google-research-datasets/mbpp",
        "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
        "license": "CC-BY-4.0",
        "url": "https://huggingface.co/datasets/google-research-datasets/mbpp/blob/4bb6404fdc6cacfda99d4ac4205087b89d32030c/sanitized/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "605",
        "pool": "proposed",
        "file_sha256": "d95f8ad6d2fff08fe4826122d6e3e31f75716825d0c5c340d297aca5e9e0de0e",
        "row_sha256": "d0410dd81bcdf05d0b85ddd3a5690547a59159092aabc923f7ba1e38353a4746",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Jacob Austin et al., Program Synthesis with Large Language Models (2021), Google Research MBPP. Adapted into repair prompts; rubrics, alternative solutions and supplemental tests added for this gallery."
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "software_repair",
        "leaf": "software_repair.functional_bug_repair",
        "confidence": "high",
        "reason": "Explicitly adapted from a function-writing task into a repair task; original prompt retained below."
      },
      "language": "en",
      "prompt": "Repair this Python implementation. For an integer n, return True exactly when n is prime (n >= 2 and has no positive divisors except 1 and itself); otherwise return False. Keep the prime_num(n) call signature.\n\n```python\ndef prime_num(num):\n  if num >=1:\n   for i in range(2, num//2):\n     if (num % i) == 0:\n                return False\n     else:\n                return True\n  else:\n          return False\n```",
      "original_prompt": "Write a function to check if the given integer is a prime number.",
      "context": [
        "Original MBPP prompt: Write a function to check if the given integer is a prime number.",
        "Task adaptation: a faulty implementation and explicit callable/behavior contract were added for this gallery. Grounding distinguishes source tests from supplemental checks."
      ],
      "environment_note": "Self-contained Python. The displayed implementations were executed locally; no external repository is required.",
      "rubric": {
        "markdown": "# Repair a reference implementation that misclassifies primes\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct behavior on source cases\n\n**ID:** source\\_behavior\n\n**Criterion:** Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** All source examples pass with the requested callable.\n- **0:** One or more source examples fail or the callable cannot be invoked.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_tests\", \"adapted\\_task\\_contract\"\\]\n\n### Generalizes across the stated contract\n\n**ID:** general\\_behavior\n\n**Criterion:** Correctly classifies integers according to the mathematical definition of a prime. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** All supplemental contract checks pass.\n- **0:** A supplemental case violates the contract.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"supplemental\\_tests\", \"adapted\\_task\\_contract\"\\]\n",
        "raw": {
          "title": "Repair a reference implementation that misclassifies primes",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "source_behavior",
              "criterion": "Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.",
              "weight": 3.0,
              "grading_anchors": {
                "1": "All source examples pass with the requested callable.",
                "0": "One or more source examples fail or the callable cannot be invoked."
              },
              "evidence_refs": [
                "source_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            },
            {
              "id": "general_behavior",
              "criterion": "Correctly classifies integers according to the mathematical definition of a prime. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "All supplemental contract checks pass.",
                "0": "A supplemental case violates the contract."
              },
              "evidence_refs": [
                "supplemental_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "source_behavior": {
            "title": "Correct behavior on source cases",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "general_behavior": {
            "title": "Generalizes across the stated contract",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "Function behavior defined by the explicit contract and tests.",
        "solution": "def prime_num(num):\n  if num >=1:\n   for i in range(2, num//2):\n     if (num % i) == 0:\n                return False\n     else:\n                return True\n  else:\n          return False",
        "tests": [
          "assert prime_num(13)==True",
          "assert prime_num(7)==True",
          "assert prime_num(-1010)==False"
        ],
        "source_reference_verified": false,
        "supplemental_test_count": 206,
        "warning": "The dataset's own reference passes all three source tests, but returns None for 2 and True for 9. The reference is evidence to audit, not an infallible oracle."
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Two different valid implementations pass the same checks.",
          "Incorrect controls expose failure modes and sparse-test blind spots.",
          "The dataset's own reference passes all three source tests, but returns None for 2 and True for 9. The reference is evidence to audit, not an infallible oracle."
        ],
        "risks": [
          "Finite tests do not prove correctness for every input.",
          "The explicit repair contract and supplemental tests are authored additions, not original source annotations."
        ],
        "must_hold": [
          "All integers below 2 are nonprime.",
          "A composite with an odd factor must be rejected even when not divisible by 2.",
          "2 and other primes must be accepted."
        ],
        "may_vary": [
          "Trial division, divisor enumeration, a sieve, or another correct algorithm.",
          "No particular loop bound, library, complexity class, or explanation is mandatory."
        ],
        "proposed_changes": [
          "Keep source reference code as one audited implementation, never as the required method.",
          "Expand behavior checks before scaling admission; use property-based and adversarial tests."
        ]
      },
      "probes": [
        {
          "label": "Trial division through the square root",
          "kind": "valid_alternative",
          "response": "```python\ndef prime_num(n):\n    if n < 2:\n        return False\n    d = 2\n    while d * d <= n:\n        if n % d == 0:\n            return False\n        d += 1\n    return True\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 206,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert prime_num(13)==True"
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert prime_num(7)==True"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert prime_num(-1010)==False"
              },
              {
                "name": "Integer -5",
                "passed": true,
                "detail": "assert prime_num(-5) == False"
              },
              {
                "name": "Integer -4",
                "passed": true,
                "detail": "assert prime_num(-4) == False"
              },
              {
                "name": "Integer -3",
                "passed": true,
                "detail": "assert prime_num(-3) == False"
              },
              {
                "name": "Integer -2",
                "passed": true,
                "detail": "assert prime_num(-2) == False"
              },
              {
                "name": "Integer -1",
                "passed": true,
                "detail": "assert prime_num(-1) == False"
              },
              {
                "name": "Integer 0",
                "passed": true,
                "detail": "assert prime_num(0) == False"
              },
              {
                "name": "Integer 1",
                "passed": true,
                "detail": "assert prime_num(1) == False"
              },
              {
                "name": "Integer 2",
                "passed": true,
                "detail": "assert prime_num(2) == True"
              },
              {
                "name": "Integer 3",
                "passed": true,
                "detail": "assert prime_num(3) == True"
              },
              {
                "name": "Integer 4",
                "passed": true,
                "detail": "assert prime_num(4) == False"
              },
              {
                "name": "Integer 5",
                "passed": true,
                "detail": "assert prime_num(5) == True"
              },
              {
                "name": "Integer 6",
                "passed": true,
                "detail": "assert prime_num(6) == False"
              },
              {
                "name": "Integer 7",
                "passed": true,
                "detail": "assert prime_num(7) == True"
              },
              {
                "name": "Integer 8",
                "passed": true,
                "detail": "assert prime_num(8) == False"
              },
              {
                "name": "Integer 9",
                "passed": true,
                "detail": "assert prime_num(9) == False"
              },
              {
                "name": "Integer 10",
                "passed": true,
                "detail": "assert prime_num(10) == False"
              },
              {
                "name": "Integer 11",
                "passed": true,
                "detail": "assert prime_num(11) == True"
              },
              {
                "name": "Integer 12",
                "passed": true,
                "detail": "assert prime_num(12) == False"
              },
              {
                "name": "Integer 13",
                "passed": true,
                "detail": "assert prime_num(13) == True"
              },
              {
                "name": "Integer 14",
                "passed": true,
                "detail": "assert prime_num(14) == False"
              },
              {
                "name": "Integer 15",
                "passed": true,
                "detail": "assert prime_num(15) == False"
              },
              {
                "name": "Integer 16",
                "passed": true,
                "detail": "assert prime_num(16) == False"
              },
              {
                "name": "Integer 17",
                "passed": true,
                "detail": "assert prime_num(17) == True"
              },
              {
                "name": "Integer 18",
                "passed": true,
                "detail": "assert prime_num(18) == False"
              },
              {
                "name": "Integer 19",
                "passed": true,
                "detail": "assert prime_num(19) == True"
              },
              {
                "name": "Integer 20",
                "passed": true,
                "detail": "assert prime_num(20) == False"
              },
              {
                "name": "Integer 21",
                "passed": true,
                "detail": "assert prime_num(21) == False"
              },
              {
                "name": "Integer 22",
                "passed": true,
                "detail": "assert prime_num(22) == False"
              },
              {
                "name": "Integer 23",
                "passed": true,
                "detail": "assert prime_num(23) == True"
              },
              {
                "name": "Integer 24",
                "passed": true,
                "detail": "assert prime_num(24) == False"
              },
              {
                "name": "Integer 25",
                "passed": true,
                "detail": "assert prime_num(25) == False"
              },
              {
                "name": "Integer 26",
                "passed": true,
                "detail": "assert prime_num(26) == False"
              },
              {
                "name": "Integer 27",
                "passed": true,
                "detail": "assert prime_num(27) == False"
              },
              {
                "name": "Integer 28",
                "passed": true,
                "detail": "assert prime_num(28) == False"
              },
              {
                "name": "Integer 29",
                "passed": true,
                "detail": "assert prime_num(29) == True"
              },
              {
                "name": "Integer 30",
                "passed": true,
                "detail": "assert prime_num(30) == False"
              },
              {
                "name": "Integer 31",
                "passed": true,
                "detail": "assert prime_num(31) == True"
              },
              {
                "name": "Integer 32",
                "passed": true,
                "detail": "assert prime_num(32) == False"
              },
              {
                "name": "Integer 33",
                "passed": true,
                "detail": "assert prime_num(33) == False"
              },
              {
                "name": "Integer 34",
                "passed": true,
                "detail": "assert prime_num(34) == False"
              },
              {
                "name": "Integer 35",
                "passed": true,
                "detail": "assert prime_num(35) == False"
              },
              {
                "name": "Integer 36",
                "passed": true,
                "detail": "assert prime_num(36) == False"
              },
              {
                "name": "Integer 37",
                "passed": true,
                "detail": "assert prime_num(37) == True"
              },
              {
                "name": "Integer 38",
                "passed": true,
                "detail": "assert prime_num(38) == False"
              },
              {
                "name": "Integer 39",
                "passed": true,
                "detail": "assert prime_num(39) == False"
              },
              {
                "name": "Integer 40",
                "passed": true,
                "detail": "assert prime_num(40) == False"
              },
              {
                "name": "Integer 41",
                "passed": true,
                "detail": "assert prime_num(41) == True"
              },
              {
                "name": "Integer 42",
                "passed": true,
                "detail": "assert prime_num(42) == False"
              },
              {
                "name": "Integer 43",
                "passed": true,
                "detail": "assert prime_num(43) == True"
              },
              {
                "name": "Integer 44",
                "passed": true,
                "detail": "assert prime_num(44) == False"
              },
              {
                "name": "Integer 45",
                "passed": true,
                "detail": "assert prime_num(45) == False"
              },
              {
                "name": "Integer 46",
                "passed": true,
                "detail": "assert prime_num(46) == False"
              },
              {
                "name": "Integer 47",
                "passed": true,
                "detail": "assert prime_num(47) == True"
              },
              {
                "name": "Integer 48",
                "passed": true,
                "detail": "assert prime_num(48) == False"
              },
              {
                "name": "Integer 49",
                "passed": true,
                "detail": "assert prime_num(49) == False"
              },
              {
                "name": "Integer 50",
                "passed": true,
                "detail": "assert prime_num(50) == False"
              },
              {
                "name": "Integer 51",
                "passed": true,
                "detail": "assert prime_num(51) == False"
              },
              {
                "name": "Integer 52",
                "passed": true,
                "detail": "assert prime_num(52) == False"
              },
              {
                "name": "Integer 53",
                "passed": true,
                "detail": "assert prime_num(53) == True"
              },
              {
                "name": "Integer 54",
                "passed": true,
                "detail": "assert prime_num(54) == False"
              },
              {
                "name": "Integer 55",
                "passed": true,
                "detail": "assert prime_num(55) == False"
              },
              {
                "name": "Integer 56",
                "passed": true,
                "detail": "assert prime_num(56) == False"
              },
              {
                "name": "Integer 57",
                "passed": true,
                "detail": "assert prime_num(57) == False"
              },
              {
                "name": "Integer 58",
                "passed": true,
                "detail": "assert prime_num(58) == False"
              },
              {
                "name": "Integer 59",
                "passed": true,
                "detail": "assert prime_num(59) == True"
              },
              {
                "name": "Integer 60",
                "passed": true,
                "detail": "assert prime_num(60) == False"
              },
              {
                "name": "Integer 61",
                "passed": true,
                "detail": "assert prime_num(61) == True"
              },
              {
                "name": "Integer 62",
                "passed": true,
                "detail": "assert prime_num(62) == False"
              },
              {
                "name": "Integer 63",
                "passed": true,
                "detail": "assert prime_num(63) == False"
              },
              {
                "name": "Integer 64",
                "passed": true,
                "detail": "assert prime_num(64) == False"
              },
              {
                "name": "Integer 65",
                "passed": true,
                "detail": "assert prime_num(65) == False"
              },
              {
                "name": "Integer 66",
                "passed": true,
                "detail": "assert prime_num(66) == False"
              },
              {
                "name": "Integer 67",
                "passed": true,
                "detail": "assert prime_num(67) == True"
              },
              {
                "name": "Integer 68",
                "passed": true,
                "detail": "assert prime_num(68) == False"
              },
              {
                "name": "Integer 69",
                "passed": true,
                "detail": "assert prime_num(69) == False"
              },
              {
                "name": "Integer 70",
                "passed": true,
                "detail": "assert prime_num(70) == False"
              },
              {
                "name": "Integer 71",
                "passed": true,
                "detail": "assert prime_num(71) == True"
              },
              {
                "name": "Integer 72",
                "passed": true,
                "detail": "assert prime_num(72) == False"
              },
              {
                "name": "Integer 73",
                "passed": true,
                "detail": "assert prime_num(73) == True"
              },
              {
                "name": "Integer 74",
                "passed": true,
                "detail": "assert prime_num(74) == False"
              },
              {
                "name": "Integer 75",
                "passed": true,
                "detail": "assert prime_num(75) == False"
              },
              {
                "name": "Integer 76",
                "passed": true,
                "detail": "assert prime_num(76) == False"
              },
              {
                "name": "Integer 77",
                "passed": true,
                "detail": "assert prime_num(77) == False"
              },
              {
                "name": "Integer 78",
                "passed": true,
                "detail": "assert prime_num(78) == False"
              },
              {
                "name": "Integer 79",
                "passed": true,
                "detail": "assert prime_num(79) == True"
              },
              {
                "name": "Integer 80",
                "passed": true,
                "detail": "assert prime_num(80) == False"
              },
              {
                "name": "Integer 81",
                "passed": true,
                "detail": "assert prime_num(81) == False"
              },
              {
                "name": "Integer 82",
                "passed": true,
                "detail": "assert prime_num(82) == False"
              },
              {
                "name": "Integer 83",
                "passed": true,
                "detail": "assert prime_num(83) == True"
              },
              {
                "name": "Integer 84",
                "passed": true,
                "detail": "assert prime_num(84) == False"
              },
              {
                "name": "Integer 85",
                "passed": true,
                "detail": "assert prime_num(85) == False"
              },
              {
                "name": "Integer 86",
                "passed": true,
                "detail": "assert prime_num(86) == False"
              },
              {
                "name": "Integer 87",
                "passed": true,
                "detail": "assert prime_num(87) == False"
              },
              {
                "name": "Integer 88",
                "passed": true,
                "detail": "assert prime_num(88) == False"
              },
              {
                "name": "Integer 89",
                "passed": true,
                "detail": "assert prime_num(89) == True"
              },
              {
                "name": "Integer 90",
                "passed": true,
                "detail": "assert prime_num(90) == False"
              },
              {
                "name": "Integer 91",
                "passed": true,
                "detail": "assert prime_num(91) == False"
              },
              {
                "name": "Integer 92",
                "passed": true,
                "detail": "assert prime_num(92) == False"
              },
              {
                "name": "Integer 93",
                "passed": true,
                "detail": "assert prime_num(93) == False"
              },
              {
                "name": "Integer 94",
                "passed": true,
                "detail": "assert prime_num(94) == False"
              },
              {
                "name": "Integer 95",
                "passed": true,
                "detail": "assert prime_num(95) == False"
              },
              {
                "name": "Integer 96",
                "passed": true,
                "detail": "assert prime_num(96) == False"
              },
              {
                "name": "Integer 97",
                "passed": true,
                "detail": "assert prime_num(97) == True"
              },
              {
                "name": "Integer 98",
                "passed": true,
                "detail": "assert prime_num(98) == False"
              },
              {
                "name": "Integer 99",
                "passed": true,
                "detail": "assert prime_num(99) == False"
              },
              {
                "name": "Integer 100",
                "passed": true,
                "detail": "assert prime_num(100) == False"
              },
              {
                "name": "Integer 101",
                "passed": true,
                "detail": "assert prime_num(101) == True"
              },
              {
                "name": "Integer 102",
                "passed": true,
                "detail": "assert prime_num(102) == False"
              },
              {
                "name": "Integer 103",
                "passed": true,
                "detail": "assert prime_num(103) == True"
              },
              {
                "name": "Integer 104",
                "passed": true,
                "detail": "assert prime_num(104) == False"
              },
              {
                "name": "Integer 105",
                "passed": true,
                "detail": "assert prime_num(105) == False"
              },
              {
                "name": "Integer 106",
                "passed": true,
                "detail": "assert prime_num(106) == False"
              },
              {
                "name": "Integer 107",
                "passed": true,
                "detail": "assert prime_num(107) == True"
              },
              {
                "name": "Integer 108",
                "passed": true,
                "detail": "assert prime_num(108) == False"
              },
              {
                "name": "Integer 109",
                "passed": true,
                "detail": "assert prime_num(109) == True"
              },
              {
                "name": "Integer 110",
                "passed": true,
                "detail": "assert prime_num(110) == False"
              },
              {
                "name": "Integer 111",
                "passed": true,
                "detail": "assert prime_num(111) == False"
              },
              {
                "name": "Integer 112",
                "passed": true,
                "detail": "assert prime_num(112) == False"
              },
              {
                "name": "Integer 113",
                "passed": true,
                "detail": "assert prime_num(113) == True"
              },
              {
                "name": "Integer 114",
                "passed": true,
                "detail": "assert prime_num(114) == False"
              },
              {
                "name": "Integer 115",
                "passed": true,
                "detail": "assert prime_num(115) == False"
              },
              {
                "name": "Integer 116",
                "passed": true,
                "detail": "assert prime_num(116) == False"
              },
              {
                "name": "Integer 117",
                "passed": true,
                "detail": "assert prime_num(117) == False"
              },
              {
                "name": "Integer 118",
                "passed": true,
                "detail": "assert prime_num(118) == False"
              },
              {
                "name": "Integer 119",
                "passed": true,
                "detail": "assert prime_num(119) == False"
              },
              {
                "name": "Integer 120",
                "passed": true,
                "detail": "assert prime_num(120) == False"
              },
              {
                "name": "Integer 121",
                "passed": true,
                "detail": "assert prime_num(121) == False"
              },
              {
                "name": "Integer 122",
                "passed": true,
                "detail": "assert prime_num(122) == False"
              },
              {
                "name": "Integer 123",
                "passed": true,
                "detail": "assert prime_num(123) == False"
              },
              {
                "name": "Integer 124",
                "passed": true,
                "detail": "assert prime_num(124) == False"
              },
              {
                "name": "Integer 125",
                "passed": true,
                "detail": "assert prime_num(125) == False"
              },
              {
                "name": "Integer 126",
                "passed": true,
                "detail": "assert prime_num(126) == False"
              },
              {
                "name": "Integer 127",
                "passed": true,
                "detail": "assert prime_num(127) == True"
              },
              {
                "name": "Integer 128",
                "passed": true,
                "detail": "assert prime_num(128) == False"
              },
              {
                "name": "Integer 129",
                "passed": true,
                "detail": "assert prime_num(129) == False"
              },
              {
                "name": "Integer 130",
                "passed": true,
                "detail": "assert prime_num(130) == False"
              },
              {
                "name": "Integer 131",
                "passed": true,
                "detail": "assert prime_num(131) == True"
              },
              {
                "name": "Integer 132",
                "passed": true,
                "detail": "assert prime_num(132) == False"
              },
              {
                "name": "Integer 133",
                "passed": true,
                "detail": "assert prime_num(133) == False"
              },
              {
                "name": "Integer 134",
                "passed": true,
                "detail": "assert prime_num(134) == False"
              },
              {
                "name": "Integer 135",
                "passed": true,
                "detail": "assert prime_num(135) == False"
              },
              {
                "name": "Integer 136",
                "passed": true,
                "detail": "assert prime_num(136) == False"
              },
              {
                "name": "Integer 137",
                "passed": true,
                "detail": "assert prime_num(137) == True"
              },
              {
                "name": "Integer 138",
                "passed": true,
                "detail": "assert prime_num(138) == False"
              },
              {
                "name": "Integer 139",
                "passed": true,
                "detail": "assert prime_num(139) == True"
              },
              {
                "name": "Integer 140",
                "passed": true,
                "detail": "assert prime_num(140) == False"
              },
              {
                "name": "Integer 141",
                "passed": true,
                "detail": "assert prime_num(141) == False"
              },
              {
                "name": "Integer 142",
                "passed": true,
                "detail": "assert prime_num(142) == False"
              },
              {
                "name": "Integer 143",
                "passed": true,
                "detail": "assert prime_num(143) == False"
              },
              {
                "name": "Integer 144",
                "passed": true,
                "detail": "assert prime_num(144) == False"
              },
              {
                "name": "Integer 145",
                "passed": true,
                "detail": "assert prime_num(145) == False"
              },
              {
                "name": "Integer 146",
                "passed": true,
                "detail": "assert prime_num(146) == False"
              },
              {
                "name": "Integer 147",
                "passed": true,
                "detail": "assert prime_num(147) == False"
              },
              {
                "name": "Integer 148",
                "passed": true,
                "detail": "assert prime_num(148) == False"
              },
              {
                "name": "Integer 149",
                "passed": true,
                "detail": "assert prime_num(149) == True"
              },
              {
                "name": "Integer 150",
                "passed": true,
                "detail": "assert prime_num(150) == False"
              },
              {
                "name": "Integer 151",
                "passed": true,
                "detail": "assert prime_num(151) == True"
              },
              {
                "name": "Integer 152",
                "passed": true,
                "detail": "assert prime_num(152) == False"
              },
              {
                "name": "Integer 153",
                "passed": true,
                "detail": "assert prime_num(153) == False"
              },
              {
                "name": "Integer 154",
                "passed": true,
                "detail": "assert prime_num(154) == False"
              },
              {
                "name": "Integer 155",
                "passed": true,
                "detail": "assert prime_num(155) == False"
              },
              {
                "name": "Integer 156",
                "passed": true,
                "detail": "assert prime_num(156) == False"
              },
              {
                "name": "Integer 157",
                "passed": true,
                "detail": "assert prime_num(157) == True"
              },
              {
                "name": "Integer 158",
                "passed": true,
                "detail": "assert prime_num(158) == False"
              },
              {
                "name": "Integer 159",
                "passed": true,
                "detail": "assert prime_num(159) == False"
              },
              {
                "name": "Integer 160",
                "passed": true,
                "detail": "assert prime_num(160) == False"
              },
              {
                "name": "Integer 161",
                "passed": true,
                "detail": "assert prime_num(161) == False"
              },
              {
                "name": "Integer 162",
                "passed": true,
                "detail": "assert prime_num(162) == False"
              },
              {
                "name": "Integer 163",
                "passed": true,
                "detail": "assert prime_num(163) == True"
              },
              {
                "name": "Integer 164",
                "passed": true,
                "detail": "assert prime_num(164) == False"
              },
              {
                "name": "Integer 165",
                "passed": true,
                "detail": "assert prime_num(165) == False"
              },
              {
                "name": "Integer 166",
                "passed": true,
                "detail": "assert prime_num(166) == False"
              },
              {
                "name": "Integer 167",
                "passed": true,
                "detail": "assert prime_num(167) == True"
              },
              {
                "name": "Integer 168",
                "passed": true,
                "detail": "assert prime_num(168) == False"
              },
              {
                "name": "Integer 169",
                "passed": true,
                "detail": "assert prime_num(169) == False"
              },
              {
                "name": "Integer 170",
                "passed": true,
                "detail": "assert prime_num(170) == False"
              },
              {
                "name": "Integer 171",
                "passed": true,
                "detail": "assert prime_num(171) == False"
              },
              {
                "name": "Integer 172",
                "passed": true,
                "detail": "assert prime_num(172) == False"
              },
              {
                "name": "Integer 173",
                "passed": true,
                "detail": "assert prime_num(173) == True"
              },
              {
                "name": "Integer 174",
                "passed": true,
                "detail": "assert prime_num(174) == False"
              },
              {
                "name": "Integer 175",
                "passed": true,
                "detail": "assert prime_num(175) == False"
              },
              {
                "name": "Integer 176",
                "passed": true,
                "detail": "assert prime_num(176) == False"
              },
              {
                "name": "Integer 177",
                "passed": true,
                "detail": "assert prime_num(177) == False"
              },
              {
                "name": "Integer 178",
                "passed": true,
                "detail": "assert prime_num(178) == False"
              },
              {
                "name": "Integer 179",
                "passed": true,
                "detail": "assert prime_num(179) == True"
              },
              {
                "name": "Integer 180",
                "passed": true,
                "detail": "assert prime_num(180) == False"
              },
              {
                "name": "Integer 181",
                "passed": true,
                "detail": "assert prime_num(181) == True"
              },
              {
                "name": "Integer 182",
                "passed": true,
                "detail": "assert prime_num(182) == False"
              },
              {
                "name": "Integer 183",
                "passed": true,
                "detail": "assert prime_num(183) == False"
              },
              {
                "name": "Integer 184",
                "passed": true,
                "detail": "assert prime_num(184) == False"
              },
              {
                "name": "Integer 185",
                "passed": true,
                "detail": "assert prime_num(185) == False"
              },
              {
                "name": "Integer 186",
                "passed": true,
                "detail": "assert prime_num(186) == False"
              },
              {
                "name": "Integer 187",
                "passed": true,
                "detail": "assert prime_num(187) == False"
              },
              {
                "name": "Integer 188",
                "passed": true,
                "detail": "assert prime_num(188) == False"
              },
              {
                "name": "Integer 189",
                "passed": true,
                "detail": "assert prime_num(189) == False"
              },
              {
                "name": "Integer 190",
                "passed": true,
                "detail": "assert prime_num(190) == False"
              },
              {
                "name": "Integer 191",
                "passed": true,
                "detail": "assert prime_num(191) == True"
              },
              {
                "name": "Integer 192",
                "passed": true,
                "detail": "assert prime_num(192) == False"
              },
              {
                "name": "Integer 193",
                "passed": true,
                "detail": "assert prime_num(193) == True"
              },
              {
                "name": "Integer 194",
                "passed": true,
                "detail": "assert prime_num(194) == False"
              },
              {
                "name": "Integer 195",
                "passed": true,
                "detail": "assert prime_num(195) == False"
              },
              {
                "name": "Integer 196",
                "passed": true,
                "detail": "assert prime_num(196) == False"
              },
              {
                "name": "Integer 197",
                "passed": true,
                "detail": "assert prime_num(197) == True"
              },
              {
                "name": "Integer 198",
                "passed": true,
                "detail": "assert prime_num(198) == False"
              },
              {
                "name": "Integer 199",
                "passed": true,
                "detail": "assert prime_num(199) == True"
              },
              {
                "name": "Integer 200",
                "passed": true,
                "detail": "assert prime_num(200) == False"
              }
            ]
          }
        },
        {
          "label": "Full divisor enumeration",
          "kind": "valid_alternative",
          "response": "```python\ndef prime_num(n):\n    return n >= 2 and all(n % d != 0 for d in range(2, n))\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 206,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert prime_num(13)==True"
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert prime_num(7)==True"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert prime_num(-1010)==False"
              },
              {
                "name": "Integer -5",
                "passed": true,
                "detail": "assert prime_num(-5) == False"
              },
              {
                "name": "Integer -4",
                "passed": true,
                "detail": "assert prime_num(-4) == False"
              },
              {
                "name": "Integer -3",
                "passed": true,
                "detail": "assert prime_num(-3) == False"
              },
              {
                "name": "Integer -2",
                "passed": true,
                "detail": "assert prime_num(-2) == False"
              },
              {
                "name": "Integer -1",
                "passed": true,
                "detail": "assert prime_num(-1) == False"
              },
              {
                "name": "Integer 0",
                "passed": true,
                "detail": "assert prime_num(0) == False"
              },
              {
                "name": "Integer 1",
                "passed": true,
                "detail": "assert prime_num(1) == False"
              },
              {
                "name": "Integer 2",
                "passed": true,
                "detail": "assert prime_num(2) == True"
              },
              {
                "name": "Integer 3",
                "passed": true,
                "detail": "assert prime_num(3) == True"
              },
              {
                "name": "Integer 4",
                "passed": true,
                "detail": "assert prime_num(4) == False"
              },
              {
                "name": "Integer 5",
                "passed": true,
                "detail": "assert prime_num(5) == True"
              },
              {
                "name": "Integer 6",
                "passed": true,
                "detail": "assert prime_num(6) == False"
              },
              {
                "name": "Integer 7",
                "passed": true,
                "detail": "assert prime_num(7) == True"
              },
              {
                "name": "Integer 8",
                "passed": true,
                "detail": "assert prime_num(8) == False"
              },
              {
                "name": "Integer 9",
                "passed": true,
                "detail": "assert prime_num(9) == False"
              },
              {
                "name": "Integer 10",
                "passed": true,
                "detail": "assert prime_num(10) == False"
              },
              {
                "name": "Integer 11",
                "passed": true,
                "detail": "assert prime_num(11) == True"
              },
              {
                "name": "Integer 12",
                "passed": true,
                "detail": "assert prime_num(12) == False"
              },
              {
                "name": "Integer 13",
                "passed": true,
                "detail": "assert prime_num(13) == True"
              },
              {
                "name": "Integer 14",
                "passed": true,
                "detail": "assert prime_num(14) == False"
              },
              {
                "name": "Integer 15",
                "passed": true,
                "detail": "assert prime_num(15) == False"
              },
              {
                "name": "Integer 16",
                "passed": true,
                "detail": "assert prime_num(16) == False"
              },
              {
                "name": "Integer 17",
                "passed": true,
                "detail": "assert prime_num(17) == True"
              },
              {
                "name": "Integer 18",
                "passed": true,
                "detail": "assert prime_num(18) == False"
              },
              {
                "name": "Integer 19",
                "passed": true,
                "detail": "assert prime_num(19) == True"
              },
              {
                "name": "Integer 20",
                "passed": true,
                "detail": "assert prime_num(20) == False"
              },
              {
                "name": "Integer 21",
                "passed": true,
                "detail": "assert prime_num(21) == False"
              },
              {
                "name": "Integer 22",
                "passed": true,
                "detail": "assert prime_num(22) == False"
              },
              {
                "name": "Integer 23",
                "passed": true,
                "detail": "assert prime_num(23) == True"
              },
              {
                "name": "Integer 24",
                "passed": true,
                "detail": "assert prime_num(24) == False"
              },
              {
                "name": "Integer 25",
                "passed": true,
                "detail": "assert prime_num(25) == False"
              },
              {
                "name": "Integer 26",
                "passed": true,
                "detail": "assert prime_num(26) == False"
              },
              {
                "name": "Integer 27",
                "passed": true,
                "detail": "assert prime_num(27) == False"
              },
              {
                "name": "Integer 28",
                "passed": true,
                "detail": "assert prime_num(28) == False"
              },
              {
                "name": "Integer 29",
                "passed": true,
                "detail": "assert prime_num(29) == True"
              },
              {
                "name": "Integer 30",
                "passed": true,
                "detail": "assert prime_num(30) == False"
              },
              {
                "name": "Integer 31",
                "passed": true,
                "detail": "assert prime_num(31) == True"
              },
              {
                "name": "Integer 32",
                "passed": true,
                "detail": "assert prime_num(32) == False"
              },
              {
                "name": "Integer 33",
                "passed": true,
                "detail": "assert prime_num(33) == False"
              },
              {
                "name": "Integer 34",
                "passed": true,
                "detail": "assert prime_num(34) == False"
              },
              {
                "name": "Integer 35",
                "passed": true,
                "detail": "assert prime_num(35) == False"
              },
              {
                "name": "Integer 36",
                "passed": true,
                "detail": "assert prime_num(36) == False"
              },
              {
                "name": "Integer 37",
                "passed": true,
                "detail": "assert prime_num(37) == True"
              },
              {
                "name": "Integer 38",
                "passed": true,
                "detail": "assert prime_num(38) == False"
              },
              {
                "name": "Integer 39",
                "passed": true,
                "detail": "assert prime_num(39) == False"
              },
              {
                "name": "Integer 40",
                "passed": true,
                "detail": "assert prime_num(40) == False"
              },
              {
                "name": "Integer 41",
                "passed": true,
                "detail": "assert prime_num(41) == True"
              },
              {
                "name": "Integer 42",
                "passed": true,
                "detail": "assert prime_num(42) == False"
              },
              {
                "name": "Integer 43",
                "passed": true,
                "detail": "assert prime_num(43) == True"
              },
              {
                "name": "Integer 44",
                "passed": true,
                "detail": "assert prime_num(44) == False"
              },
              {
                "name": "Integer 45",
                "passed": true,
                "detail": "assert prime_num(45) == False"
              },
              {
                "name": "Integer 46",
                "passed": true,
                "detail": "assert prime_num(46) == False"
              },
              {
                "name": "Integer 47",
                "passed": true,
                "detail": "assert prime_num(47) == True"
              },
              {
                "name": "Integer 48",
                "passed": true,
                "detail": "assert prime_num(48) == False"
              },
              {
                "name": "Integer 49",
                "passed": true,
                "detail": "assert prime_num(49) == False"
              },
              {
                "name": "Integer 50",
                "passed": true,
                "detail": "assert prime_num(50) == False"
              },
              {
                "name": "Integer 51",
                "passed": true,
                "detail": "assert prime_num(51) == False"
              },
              {
                "name": "Integer 52",
                "passed": true,
                "detail": "assert prime_num(52) == False"
              },
              {
                "name": "Integer 53",
                "passed": true,
                "detail": "assert prime_num(53) == True"
              },
              {
                "name": "Integer 54",
                "passed": true,
                "detail": "assert prime_num(54) == False"
              },
              {
                "name": "Integer 55",
                "passed": true,
                "detail": "assert prime_num(55) == False"
              },
              {
                "name": "Integer 56",
                "passed": true,
                "detail": "assert prime_num(56) == False"
              },
              {
                "name": "Integer 57",
                "passed": true,
                "detail": "assert prime_num(57) == False"
              },
              {
                "name": "Integer 58",
                "passed": true,
                "detail": "assert prime_num(58) == False"
              },
              {
                "name": "Integer 59",
                "passed": true,
                "detail": "assert prime_num(59) == True"
              },
              {
                "name": "Integer 60",
                "passed": true,
                "detail": "assert prime_num(60) == False"
              },
              {
                "name": "Integer 61",
                "passed": true,
                "detail": "assert prime_num(61) == True"
              },
              {
                "name": "Integer 62",
                "passed": true,
                "detail": "assert prime_num(62) == False"
              },
              {
                "name": "Integer 63",
                "passed": true,
                "detail": "assert prime_num(63) == False"
              },
              {
                "name": "Integer 64",
                "passed": true,
                "detail": "assert prime_num(64) == False"
              },
              {
                "name": "Integer 65",
                "passed": true,
                "detail": "assert prime_num(65) == False"
              },
              {
                "name": "Integer 66",
                "passed": true,
                "detail": "assert prime_num(66) == False"
              },
              {
                "name": "Integer 67",
                "passed": true,
                "detail": "assert prime_num(67) == True"
              },
              {
                "name": "Integer 68",
                "passed": true,
                "detail": "assert prime_num(68) == False"
              },
              {
                "name": "Integer 69",
                "passed": true,
                "detail": "assert prime_num(69) == False"
              },
              {
                "name": "Integer 70",
                "passed": true,
                "detail": "assert prime_num(70) == False"
              },
              {
                "name": "Integer 71",
                "passed": true,
                "detail": "assert prime_num(71) == True"
              },
              {
                "name": "Integer 72",
                "passed": true,
                "detail": "assert prime_num(72) == False"
              },
              {
                "name": "Integer 73",
                "passed": true,
                "detail": "assert prime_num(73) == True"
              },
              {
                "name": "Integer 74",
                "passed": true,
                "detail": "assert prime_num(74) == False"
              },
              {
                "name": "Integer 75",
                "passed": true,
                "detail": "assert prime_num(75) == False"
              },
              {
                "name": "Integer 76",
                "passed": true,
                "detail": "assert prime_num(76) == False"
              },
              {
                "name": "Integer 77",
                "passed": true,
                "detail": "assert prime_num(77) == False"
              },
              {
                "name": "Integer 78",
                "passed": true,
                "detail": "assert prime_num(78) == False"
              },
              {
                "name": "Integer 79",
                "passed": true,
                "detail": "assert prime_num(79) == True"
              },
              {
                "name": "Integer 80",
                "passed": true,
                "detail": "assert prime_num(80) == False"
              },
              {
                "name": "Integer 81",
                "passed": true,
                "detail": "assert prime_num(81) == False"
              },
              {
                "name": "Integer 82",
                "passed": true,
                "detail": "assert prime_num(82) == False"
              },
              {
                "name": "Integer 83",
                "passed": true,
                "detail": "assert prime_num(83) == True"
              },
              {
                "name": "Integer 84",
                "passed": true,
                "detail": "assert prime_num(84) == False"
              },
              {
                "name": "Integer 85",
                "passed": true,
                "detail": "assert prime_num(85) == False"
              },
              {
                "name": "Integer 86",
                "passed": true,
                "detail": "assert prime_num(86) == False"
              },
              {
                "name": "Integer 87",
                "passed": true,
                "detail": "assert prime_num(87) == False"
              },
              {
                "name": "Integer 88",
                "passed": true,
                "detail": "assert prime_num(88) == False"
              },
              {
                "name": "Integer 89",
                "passed": true,
                "detail": "assert prime_num(89) == True"
              },
              {
                "name": "Integer 90",
                "passed": true,
                "detail": "assert prime_num(90) == False"
              },
              {
                "name": "Integer 91",
                "passed": true,
                "detail": "assert prime_num(91) == False"
              },
              {
                "name": "Integer 92",
                "passed": true,
                "detail": "assert prime_num(92) == False"
              },
              {
                "name": "Integer 93",
                "passed": true,
                "detail": "assert prime_num(93) == False"
              },
              {
                "name": "Integer 94",
                "passed": true,
                "detail": "assert prime_num(94) == False"
              },
              {
                "name": "Integer 95",
                "passed": true,
                "detail": "assert prime_num(95) == False"
              },
              {
                "name": "Integer 96",
                "passed": true,
                "detail": "assert prime_num(96) == False"
              },
              {
                "name": "Integer 97",
                "passed": true,
                "detail": "assert prime_num(97) == True"
              },
              {
                "name": "Integer 98",
                "passed": true,
                "detail": "assert prime_num(98) == False"
              },
              {
                "name": "Integer 99",
                "passed": true,
                "detail": "assert prime_num(99) == False"
              },
              {
                "name": "Integer 100",
                "passed": true,
                "detail": "assert prime_num(100) == False"
              },
              {
                "name": "Integer 101",
                "passed": true,
                "detail": "assert prime_num(101) == True"
              },
              {
                "name": "Integer 102",
                "passed": true,
                "detail": "assert prime_num(102) == False"
              },
              {
                "name": "Integer 103",
                "passed": true,
                "detail": "assert prime_num(103) == True"
              },
              {
                "name": "Integer 104",
                "passed": true,
                "detail": "assert prime_num(104) == False"
              },
              {
                "name": "Integer 105",
                "passed": true,
                "detail": "assert prime_num(105) == False"
              },
              {
                "name": "Integer 106",
                "passed": true,
                "detail": "assert prime_num(106) == False"
              },
              {
                "name": "Integer 107",
                "passed": true,
                "detail": "assert prime_num(107) == True"
              },
              {
                "name": "Integer 108",
                "passed": true,
                "detail": "assert prime_num(108) == False"
              },
              {
                "name": "Integer 109",
                "passed": true,
                "detail": "assert prime_num(109) == True"
              },
              {
                "name": "Integer 110",
                "passed": true,
                "detail": "assert prime_num(110) == False"
              },
              {
                "name": "Integer 111",
                "passed": true,
                "detail": "assert prime_num(111) == False"
              },
              {
                "name": "Integer 112",
                "passed": true,
                "detail": "assert prime_num(112) == False"
              },
              {
                "name": "Integer 113",
                "passed": true,
                "detail": "assert prime_num(113) == True"
              },
              {
                "name": "Integer 114",
                "passed": true,
                "detail": "assert prime_num(114) == False"
              },
              {
                "name": "Integer 115",
                "passed": true,
                "detail": "assert prime_num(115) == False"
              },
              {
                "name": "Integer 116",
                "passed": true,
                "detail": "assert prime_num(116) == False"
              },
              {
                "name": "Integer 117",
                "passed": true,
                "detail": "assert prime_num(117) == False"
              },
              {
                "name": "Integer 118",
                "passed": true,
                "detail": "assert prime_num(118) == False"
              },
              {
                "name": "Integer 119",
                "passed": true,
                "detail": "assert prime_num(119) == False"
              },
              {
                "name": "Integer 120",
                "passed": true,
                "detail": "assert prime_num(120) == False"
              },
              {
                "name": "Integer 121",
                "passed": true,
                "detail": "assert prime_num(121) == False"
              },
              {
                "name": "Integer 122",
                "passed": true,
                "detail": "assert prime_num(122) == False"
              },
              {
                "name": "Integer 123",
                "passed": true,
                "detail": "assert prime_num(123) == False"
              },
              {
                "name": "Integer 124",
                "passed": true,
                "detail": "assert prime_num(124) == False"
              },
              {
                "name": "Integer 125",
                "passed": true,
                "detail": "assert prime_num(125) == False"
              },
              {
                "name": "Integer 126",
                "passed": true,
                "detail": "assert prime_num(126) == False"
              },
              {
                "name": "Integer 127",
                "passed": true,
                "detail": "assert prime_num(127) == True"
              },
              {
                "name": "Integer 128",
                "passed": true,
                "detail": "assert prime_num(128) == False"
              },
              {
                "name": "Integer 129",
                "passed": true,
                "detail": "assert prime_num(129) == False"
              },
              {
                "name": "Integer 130",
                "passed": true,
                "detail": "assert prime_num(130) == False"
              },
              {
                "name": "Integer 131",
                "passed": true,
                "detail": "assert prime_num(131) == True"
              },
              {
                "name": "Integer 132",
                "passed": true,
                "detail": "assert prime_num(132) == False"
              },
              {
                "name": "Integer 133",
                "passed": true,
                "detail": "assert prime_num(133) == False"
              },
              {
                "name": "Integer 134",
                "passed": true,
                "detail": "assert prime_num(134) == False"
              },
              {
                "name": "Integer 135",
                "passed": true,
                "detail": "assert prime_num(135) == False"
              },
              {
                "name": "Integer 136",
                "passed": true,
                "detail": "assert prime_num(136) == False"
              },
              {
                "name": "Integer 137",
                "passed": true,
                "detail": "assert prime_num(137) == True"
              },
              {
                "name": "Integer 138",
                "passed": true,
                "detail": "assert prime_num(138) == False"
              },
              {
                "name": "Integer 139",
                "passed": true,
                "detail": "assert prime_num(139) == True"
              },
              {
                "name": "Integer 140",
                "passed": true,
                "detail": "assert prime_num(140) == False"
              },
              {
                "name": "Integer 141",
                "passed": true,
                "detail": "assert prime_num(141) == False"
              },
              {
                "name": "Integer 142",
                "passed": true,
                "detail": "assert prime_num(142) == False"
              },
              {
                "name": "Integer 143",
                "passed": true,
                "detail": "assert prime_num(143) == False"
              },
              {
                "name": "Integer 144",
                "passed": true,
                "detail": "assert prime_num(144) == False"
              },
              {
                "name": "Integer 145",
                "passed": true,
                "detail": "assert prime_num(145) == False"
              },
              {
                "name": "Integer 146",
                "passed": true,
                "detail": "assert prime_num(146) == False"
              },
              {
                "name": "Integer 147",
                "passed": true,
                "detail": "assert prime_num(147) == False"
              },
              {
                "name": "Integer 148",
                "passed": true,
                "detail": "assert prime_num(148) == False"
              },
              {
                "name": "Integer 149",
                "passed": true,
                "detail": "assert prime_num(149) == True"
              },
              {
                "name": "Integer 150",
                "passed": true,
                "detail": "assert prime_num(150) == False"
              },
              {
                "name": "Integer 151",
                "passed": true,
                "detail": "assert prime_num(151) == True"
              },
              {
                "name": "Integer 152",
                "passed": true,
                "detail": "assert prime_num(152) == False"
              },
              {
                "name": "Integer 153",
                "passed": true,
                "detail": "assert prime_num(153) == False"
              },
              {
                "name": "Integer 154",
                "passed": true,
                "detail": "assert prime_num(154) == False"
              },
              {
                "name": "Integer 155",
                "passed": true,
                "detail": "assert prime_num(155) == False"
              },
              {
                "name": "Integer 156",
                "passed": true,
                "detail": "assert prime_num(156) == False"
              },
              {
                "name": "Integer 157",
                "passed": true,
                "detail": "assert prime_num(157) == True"
              },
              {
                "name": "Integer 158",
                "passed": true,
                "detail": "assert prime_num(158) == False"
              },
              {
                "name": "Integer 159",
                "passed": true,
                "detail": "assert prime_num(159) == False"
              },
              {
                "name": "Integer 160",
                "passed": true,
                "detail": "assert prime_num(160) == False"
              },
              {
                "name": "Integer 161",
                "passed": true,
                "detail": "assert prime_num(161) == False"
              },
              {
                "name": "Integer 162",
                "passed": true,
                "detail": "assert prime_num(162) == False"
              },
              {
                "name": "Integer 163",
                "passed": true,
                "detail": "assert prime_num(163) == True"
              },
              {
                "name": "Integer 164",
                "passed": true,
                "detail": "assert prime_num(164) == False"
              },
              {
                "name": "Integer 165",
                "passed": true,
                "detail": "assert prime_num(165) == False"
              },
              {
                "name": "Integer 166",
                "passed": true,
                "detail": "assert prime_num(166) == False"
              },
              {
                "name": "Integer 167",
                "passed": true,
                "detail": "assert prime_num(167) == True"
              },
              {
                "name": "Integer 168",
                "passed": true,
                "detail": "assert prime_num(168) == False"
              },
              {
                "name": "Integer 169",
                "passed": true,
                "detail": "assert prime_num(169) == False"
              },
              {
                "name": "Integer 170",
                "passed": true,
                "detail": "assert prime_num(170) == False"
              },
              {
                "name": "Integer 171",
                "passed": true,
                "detail": "assert prime_num(171) == False"
              },
              {
                "name": "Integer 172",
                "passed": true,
                "detail": "assert prime_num(172) == False"
              },
              {
                "name": "Integer 173",
                "passed": true,
                "detail": "assert prime_num(173) == True"
              },
              {
                "name": "Integer 174",
                "passed": true,
                "detail": "assert prime_num(174) == False"
              },
              {
                "name": "Integer 175",
                "passed": true,
                "detail": "assert prime_num(175) == False"
              },
              {
                "name": "Integer 176",
                "passed": true,
                "detail": "assert prime_num(176) == False"
              },
              {
                "name": "Integer 177",
                "passed": true,
                "detail": "assert prime_num(177) == False"
              },
              {
                "name": "Integer 178",
                "passed": true,
                "detail": "assert prime_num(178) == False"
              },
              {
                "name": "Integer 179",
                "passed": true,
                "detail": "assert prime_num(179) == True"
              },
              {
                "name": "Integer 180",
                "passed": true,
                "detail": "assert prime_num(180) == False"
              },
              {
                "name": "Integer 181",
                "passed": true,
                "detail": "assert prime_num(181) == True"
              },
              {
                "name": "Integer 182",
                "passed": true,
                "detail": "assert prime_num(182) == False"
              },
              {
                "name": "Integer 183",
                "passed": true,
                "detail": "assert prime_num(183) == False"
              },
              {
                "name": "Integer 184",
                "passed": true,
                "detail": "assert prime_num(184) == False"
              },
              {
                "name": "Integer 185",
                "passed": true,
                "detail": "assert prime_num(185) == False"
              },
              {
                "name": "Integer 186",
                "passed": true,
                "detail": "assert prime_num(186) == False"
              },
              {
                "name": "Integer 187",
                "passed": true,
                "detail": "assert prime_num(187) == False"
              },
              {
                "name": "Integer 188",
                "passed": true,
                "detail": "assert prime_num(188) == False"
              },
              {
                "name": "Integer 189",
                "passed": true,
                "detail": "assert prime_num(189) == False"
              },
              {
                "name": "Integer 190",
                "passed": true,
                "detail": "assert prime_num(190) == False"
              },
              {
                "name": "Integer 191",
                "passed": true,
                "detail": "assert prime_num(191) == True"
              },
              {
                "name": "Integer 192",
                "passed": true,
                "detail": "assert prime_num(192) == False"
              },
              {
                "name": "Integer 193",
                "passed": true,
                "detail": "assert prime_num(193) == True"
              },
              {
                "name": "Integer 194",
                "passed": true,
                "detail": "assert prime_num(194) == False"
              },
              {
                "name": "Integer 195",
                "passed": true,
                "detail": "assert prime_num(195) == False"
              },
              {
                "name": "Integer 196",
                "passed": true,
                "detail": "assert prime_num(196) == False"
              },
              {
                "name": "Integer 197",
                "passed": true,
                "detail": "assert prime_num(197) == True"
              },
              {
                "name": "Integer 198",
                "passed": true,
                "detail": "assert prime_num(198) == False"
              },
              {
                "name": "Integer 199",
                "passed": true,
                "detail": "assert prime_num(199) == True"
              },
              {
                "name": "Integer 200",
                "passed": true,
                "detail": "assert prime_num(200) == False"
              }
            ]
          }
        },
        {
          "label": "Faulty source reference",
          "kind": "invalid_control",
          "response": "```python\ndef prime_num(num):\n  if num >=1:\n   for i in range(2, num//2):\n     if (num % i) == 0:\n                return False\n     else:\n                return True\n  else:\n          return False\n```",
          "is_full_response": true,
          "expected": "Reject: violates the behavioral contract.",
          "verification": "executed",
          "why": "The dataset's own reference passes all three source tests, but returns None for 2 and True for 9. The reference is evidence to audit, not an infallible oracle.",
          "result": {
            "passed": false,
            "failure_count": 59,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert prime_num(13)==True"
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert prime_num(7)==True"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert prime_num(-1010)==False"
              },
              {
                "name": "Integer -5",
                "passed": true,
                "detail": "assert prime_num(-5) == False"
              },
              {
                "name": "Integer -4",
                "passed": true,
                "detail": "assert prime_num(-4) == False"
              },
              {
                "name": "Integer -3",
                "passed": true,
                "detail": "assert prime_num(-3) == False"
              },
              {
                "name": "Integer -2",
                "passed": true,
                "detail": "assert prime_num(-2) == False"
              },
              {
                "name": "Integer -1",
                "passed": true,
                "detail": "assert prime_num(-1) == False"
              },
              {
                "name": "Integer 0",
                "passed": true,
                "detail": "assert prime_num(0) == False"
              },
              {
                "name": "Integer 1",
                "passed": false,
                "detail": "assert prime_num(1) == False -> AssertionError"
              },
              {
                "name": "Integer 2",
                "passed": false,
                "detail": "assert prime_num(2) == True -> AssertionError"
              },
              {
                "name": "Integer 3",
                "passed": false,
                "detail": "assert prime_num(3) == True -> AssertionError"
              },
              {
                "name": "Integer 4",
                "passed": false,
                "detail": "assert prime_num(4) == False -> AssertionError"
              },
              {
                "name": "Integer 5",
                "passed": false,
                "detail": "assert prime_num(5) == True -> AssertionError"
              },
              {
                "name": "Integer 6",
                "passed": true,
                "detail": "assert prime_num(6) == False"
              },
              {
                "name": "Integer 7",
                "passed": true,
                "detail": "assert prime_num(7) == True"
              },
              {
                "name": "Integer 8",
                "passed": true,
                "detail": "assert prime_num(8) == False"
              },
              {
                "name": "Integer 9",
                "passed": false,
                "detail": "assert prime_num(9) == False -> AssertionError"
              },
              {
                "name": "Integer 10",
                "passed": true,
                "detail": "assert prime_num(10) == False"
              },
              {
                "name": "Integer 11",
                "passed": true,
                "detail": "assert prime_num(11) == True"
              },
              {
                "name": "Integer 12",
                "passed": true,
                "detail": "assert prime_num(12) == False"
              },
              {
                "name": "Integer 13",
                "passed": true,
                "detail": "assert prime_num(13) == True"
              },
              {
                "name": "Integer 14",
                "passed": true,
                "detail": "assert prime_num(14) == False"
              },
              {
                "name": "Integer 15",
                "passed": false,
                "detail": "assert prime_num(15) == False -> AssertionError"
              },
              {
                "name": "Integer 16",
                "passed": true,
                "detail": "assert prime_num(16) == False"
              },
              {
                "name": "Integer 17",
                "passed": true,
                "detail": "assert prime_num(17) == True"
              },
              {
                "name": "Integer 18",
                "passed": true,
                "detail": "assert prime_num(18) == False"
              },
              {
                "name": "Integer 19",
                "passed": true,
                "detail": "assert prime_num(19) == True"
              },
              {
                "name": "Integer 20",
                "passed": true,
                "detail": "assert prime_num(20) == False"
              },
              {
                "name": "Integer 21",
                "passed": false,
                "detail": "assert prime_num(21) == False -> AssertionError"
              },
              {
                "name": "Integer 22",
                "passed": true,
                "detail": "assert prime_num(22) == False"
              },
              {
                "name": "Integer 23",
                "passed": true,
                "detail": "assert prime_num(23) == True"
              },
              {
                "name": "Integer 24",
                "passed": true,
                "detail": "assert prime_num(24) == False"
              },
              {
                "name": "Integer 25",
                "passed": false,
                "detail": "assert prime_num(25) == False -> AssertionError"
              },
              {
                "name": "Integer 26",
                "passed": true,
                "detail": "assert prime_num(26) == False"
              },
              {
                "name": "Integer 27",
                "passed": false,
                "detail": "assert prime_num(27) == False -> AssertionError"
              },
              {
                "name": "Integer 28",
                "passed": true,
                "detail": "assert prime_num(28) == False"
              },
              {
                "name": "Integer 29",
                "passed": true,
                "detail": "assert prime_num(29) == True"
              },
              {
                "name": "Integer 30",
                "passed": true,
                "detail": "assert prime_num(30) == False"
              },
              {
                "name": "Integer 31",
                "passed": true,
                "detail": "assert prime_num(31) == True"
              },
              {
                "name": "Integer 32",
                "passed": true,
                "detail": "assert prime_num(32) == False"
              },
              {
                "name": "Integer 33",
                "passed": false,
                "detail": "assert prime_num(33) == False -> AssertionError"
              },
              {
                "name": "Integer 34",
                "passed": true,
                "detail": "assert prime_num(34) == False"
              },
              {
                "name": "Integer 35",
                "passed": false,
                "detail": "assert prime_num(35) == False -> AssertionError"
              },
              {
                "name": "Integer 36",
                "passed": true,
                "detail": "assert prime_num(36) == False"
              },
              {
                "name": "Integer 37",
                "passed": true,
                "detail": "assert prime_num(37) == True"
              },
              {
                "name": "Integer 38",
                "passed": true,
                "detail": "assert prime_num(38) == False"
              },
              {
                "name": "Integer 39",
                "passed": false,
                "detail": "assert prime_num(39) == False -> AssertionError"
              },
              {
                "name": "Integer 40",
                "passed": true,
                "detail": "assert prime_num(40) == False"
              },
              {
                "name": "Integer 41",
                "passed": true,
                "detail": "assert prime_num(41) == True"
              },
              {
                "name": "Integer 42",
                "passed": true,
                "detail": "assert prime_num(42) == False"
              },
              {
                "name": "Integer 43",
                "passed": true,
                "detail": "assert prime_num(43) == True"
              },
              {
                "name": "Integer 44",
                "passed": true,
                "detail": "assert prime_num(44) == False"
              },
              {
                "name": "Integer 45",
                "passed": false,
                "detail": "assert prime_num(45) == False -> AssertionError"
              },
              {
                "name": "Integer 46",
                "passed": true,
                "detail": "assert prime_num(46) == False"
              },
              {
                "name": "Integer 47",
                "passed": true,
                "detail": "assert prime_num(47) == True"
              },
              {
                "name": "Integer 48",
                "passed": true,
                "detail": "assert prime_num(48) == False"
              },
              {
                "name": "Integer 49",
                "passed": false,
                "detail": "assert prime_num(49) == False -> AssertionError"
              },
              {
                "name": "Integer 50",
                "passed": true,
                "detail": "assert prime_num(50) == False"
              },
              {
                "name": "Integer 51",
                "passed": false,
                "detail": "assert prime_num(51) == False -> AssertionError"
              },
              {
                "name": "Integer 52",
                "passed": true,
                "detail": "assert prime_num(52) == False"
              },
              {
                "name": "Integer 53",
                "passed": true,
                "detail": "assert prime_num(53) == True"
              },
              {
                "name": "Integer 54",
                "passed": true,
                "detail": "assert prime_num(54) == False"
              },
              {
                "name": "Integer 55",
                "passed": false,
                "detail": "assert prime_num(55) == False -> AssertionError"
              },
              {
                "name": "Integer 56",
                "passed": true,
                "detail": "assert prime_num(56) == False"
              },
              {
                "name": "Integer 57",
                "passed": false,
                "detail": "assert prime_num(57) == False -> AssertionError"
              },
              {
                "name": "Integer 58",
                "passed": true,
                "detail": "assert prime_num(58) == False"
              },
              {
                "name": "Integer 59",
                "passed": true,
                "detail": "assert prime_num(59) == True"
              },
              {
                "name": "Integer 60",
                "passed": true,
                "detail": "assert prime_num(60) == False"
              },
              {
                "name": "Integer 61",
                "passed": true,
                "detail": "assert prime_num(61) == True"
              },
              {
                "name": "Integer 62",
                "passed": true,
                "detail": "assert prime_num(62) == False"
              },
              {
                "name": "Integer 63",
                "passed": false,
                "detail": "assert prime_num(63) == False -> AssertionError"
              },
              {
                "name": "Integer 64",
                "passed": true,
                "detail": "assert prime_num(64) == False"
              },
              {
                "name": "Integer 65",
                "passed": false,
                "detail": "assert prime_num(65) == False -> AssertionError"
              },
              {
                "name": "Integer 66",
                "passed": true,
                "detail": "assert prime_num(66) == False"
              },
              {
                "name": "Integer 67",
                "passed": true,
                "detail": "assert prime_num(67) == True"
              },
              {
                "name": "Integer 68",
                "passed": true,
                "detail": "assert prime_num(68) == False"
              },
              {
                "name": "Integer 69",
                "passed": false,
                "detail": "assert prime_num(69) == False -> AssertionError"
              },
              {
                "name": "Integer 70",
                "passed": true,
                "detail": "assert prime_num(70) == False"
              },
              {
                "name": "Integer 71",
                "passed": true,
                "detail": "assert prime_num(71) == True"
              },
              {
                "name": "Integer 72",
                "passed": true,
                "detail": "assert prime_num(72) == False"
              },
              {
                "name": "Integer 73",
                "passed": true,
                "detail": "assert prime_num(73) == True"
              },
              {
                "name": "Integer 74",
                "passed": true,
                "detail": "assert prime_num(74) == False"
              },
              {
                "name": "Integer 75",
                "passed": false,
                "detail": "assert prime_num(75) == False -> AssertionError"
              },
              {
                "name": "Integer 76",
                "passed": true,
                "detail": "assert prime_num(76) == False"
              },
              {
                "name": "Integer 77",
                "passed": false,
                "detail": "assert prime_num(77) == False -> AssertionError"
              },
              {
                "name": "Integer 78",
                "passed": true,
                "detail": "assert prime_num(78) == False"
              },
              {
                "name": "Integer 79",
                "passed": true,
                "detail": "assert prime_num(79) == True"
              },
              {
                "name": "Integer 80",
                "passed": true,
                "detail": "assert prime_num(80) == False"
              },
              {
                "name": "Integer 81",
                "passed": false,
                "detail": "assert prime_num(81) == False -> AssertionError"
              },
              {
                "name": "Integer 82",
                "passed": true,
                "detail": "assert prime_num(82) == False"
              },
              {
                "name": "Integer 83",
                "passed": true,
                "detail": "assert prime_num(83) == True"
              },
              {
                "name": "Integer 84",
                "passed": true,
                "detail": "assert prime_num(84) == False"
              },
              {
                "name": "Integer 85",
                "passed": false,
                "detail": "assert prime_num(85) == False -> AssertionError"
              },
              {
                "name": "Integer 86",
                "passed": true,
                "detail": "assert prime_num(86) == False"
              },
              {
                "name": "Integer 87",
                "passed": false,
                "detail": "assert prime_num(87) == False -> AssertionError"
              },
              {
                "name": "Integer 88",
                "passed": true,
                "detail": "assert prime_num(88) == False"
              },
              {
                "name": "Integer 89",
                "passed": true,
                "detail": "assert prime_num(89) == True"
              },
              {
                "name": "Integer 90",
                "passed": true,
                "detail": "assert prime_num(90) == False"
              },
              {
                "name": "Integer 91",
                "passed": false,
                "detail": "assert prime_num(91) == False -> AssertionError"
              },
              {
                "name": "Integer 92",
                "passed": true,
                "detail": "assert prime_num(92) == False"
              },
              {
                "name": "Integer 93",
                "passed": false,
                "detail": "assert prime_num(93) == False -> AssertionError"
              },
              {
                "name": "Integer 94",
                "passed": true,
                "detail": "assert prime_num(94) == False"
              },
              {
                "name": "Integer 95",
                "passed": false,
                "detail": "assert prime_num(95) == False -> AssertionError"
              },
              {
                "name": "Integer 96",
                "passed": true,
                "detail": "assert prime_num(96) == False"
              },
              {
                "name": "Integer 97",
                "passed": true,
                "detail": "assert prime_num(97) == True"
              },
              {
                "name": "Integer 98",
                "passed": true,
                "detail": "assert prime_num(98) == False"
              },
              {
                "name": "Integer 99",
                "passed": false,
                "detail": "assert prime_num(99) == False -> AssertionError"
              },
              {
                "name": "Integer 100",
                "passed": true,
                "detail": "assert prime_num(100) == False"
              },
              {
                "name": "Integer 101",
                "passed": true,
                "detail": "assert prime_num(101) == True"
              },
              {
                "name": "Integer 102",
                "passed": true,
                "detail": "assert prime_num(102) == False"
              },
              {
                "name": "Integer 103",
                "passed": true,
                "detail": "assert prime_num(103) == True"
              },
              {
                "name": "Integer 104",
                "passed": true,
                "detail": "assert prime_num(104) == False"
              },
              {
                "name": "Integer 105",
                "passed": false,
                "detail": "assert prime_num(105) == False -> AssertionError"
              },
              {
                "name": "Integer 106",
                "passed": true,
                "detail": "assert prime_num(106) == False"
              },
              {
                "name": "Integer 107",
                "passed": true,
                "detail": "assert prime_num(107) == True"
              },
              {
                "name": "Integer 108",
                "passed": true,
                "detail": "assert prime_num(108) == False"
              },
              {
                "name": "Integer 109",
                "passed": true,
                "detail": "assert prime_num(109) == True"
              },
              {
                "name": "Integer 110",
                "passed": true,
                "detail": "assert prime_num(110) == False"
              },
              {
                "name": "Integer 111",
                "passed": false,
                "detail": "assert prime_num(111) == False -> AssertionError"
              },
              {
                "name": "Integer 112",
                "passed": true,
                "detail": "assert prime_num(112) == False"
              },
              {
                "name": "Integer 113",
                "passed": true,
                "detail": "assert prime_num(113) == True"
              },
              {
                "name": "Integer 114",
                "passed": true,
                "detail": "assert prime_num(114) == False"
              },
              {
                "name": "Integer 115",
                "passed": false,
                "detail": "assert prime_num(115) == False -> AssertionError"
              },
              {
                "name": "Integer 116",
                "passed": true,
                "detail": "assert prime_num(116) == False"
              },
              {
                "name": "Integer 117",
                "passed": false,
                "detail": "assert prime_num(117) == False -> AssertionError"
              },
              {
                "name": "Integer 118",
                "passed": true,
                "detail": "assert prime_num(118) == False"
              },
              {
                "name": "Integer 119",
                "passed": false,
                "detail": "assert prime_num(119) == False -> AssertionError"
              },
              {
                "name": "Integer 120",
                "passed": true,
                "detail": "assert prime_num(120) == False"
              },
              {
                "name": "Integer 121",
                "passed": false,
                "detail": "assert prime_num(121) == False -> AssertionError"
              },
              {
                "name": "Integer 122",
                "passed": true,
                "detail": "assert prime_num(122) == False"
              },
              {
                "name": "Integer 123",
                "passed": false,
                "detail": "assert prime_num(123) == False -> AssertionError"
              },
              {
                "name": "Integer 124",
                "passed": true,
                "detail": "assert prime_num(124) == False"
              },
              {
                "name": "Integer 125",
                "passed": false,
                "detail": "assert prime_num(125) == False -> AssertionError"
              },
              {
                "name": "Integer 126",
                "passed": true,
                "detail": "assert prime_num(126) == False"
              },
              {
                "name": "Integer 127",
                "passed": true,
                "detail": "assert prime_num(127) == True"
              },
              {
                "name": "Integer 128",
                "passed": true,
                "detail": "assert prime_num(128) == False"
              },
              {
                "name": "Integer 129",
                "passed": false,
                "detail": "assert prime_num(129) == False -> AssertionError"
              },
              {
                "name": "Integer 130",
                "passed": true,
                "detail": "assert prime_num(130) == False"
              },
              {
                "name": "Integer 131",
                "passed": true,
                "detail": "assert prime_num(131) == True"
              },
              {
                "name": "Integer 132",
                "passed": true,
                "detail": "assert prime_num(132) == False"
              },
              {
                "name": "Integer 133",
                "passed": false,
                "detail": "assert prime_num(133) == False -> AssertionError"
              },
              {
                "name": "Integer 134",
                "passed": true,
                "detail": "assert prime_num(134) == False"
              },
              {
                "name": "Integer 135",
                "passed": false,
                "detail": "assert prime_num(135) == False -> AssertionError"
              },
              {
                "name": "Integer 136",
                "passed": true,
                "detail": "assert prime_num(136) == False"
              },
              {
                "name": "Integer 137",
                "passed": true,
                "detail": "assert prime_num(137) == True"
              },
              {
                "name": "Integer 138",
                "passed": true,
                "detail": "assert prime_num(138) == False"
              },
              {
                "name": "Integer 139",
                "passed": true,
                "detail": "assert prime_num(139) == True"
              },
              {
                "name": "Integer 140",
                "passed": true,
                "detail": "assert prime_num(140) == False"
              },
              {
                "name": "Integer 141",
                "passed": false,
                "detail": "assert prime_num(141) == False -> AssertionError"
              },
              {
                "name": "Integer 142",
                "passed": true,
                "detail": "assert prime_num(142) == False"
              },
              {
                "name": "Integer 143",
                "passed": false,
                "detail": "assert prime_num(143) == False -> AssertionError"
              },
              {
                "name": "Integer 144",
                "passed": true,
                "detail": "assert prime_num(144) == False"
              },
              {
                "name": "Integer 145",
                "passed": false,
                "detail": "assert prime_num(145) == False -> AssertionError"
              },
              {
                "name": "Integer 146",
                "passed": true,
                "detail": "assert prime_num(146) == False"
              },
              {
                "name": "Integer 147",
                "passed": false,
                "detail": "assert prime_num(147) == False -> AssertionError"
              },
              {
                "name": "Integer 148",
                "passed": true,
                "detail": "assert prime_num(148) == False"
              },
              {
                "name": "Integer 149",
                "passed": true,
                "detail": "assert prime_num(149) == True"
              },
              {
                "name": "Integer 150",
                "passed": true,
                "detail": "assert prime_num(150) == False"
              },
              {
                "name": "Integer 151",
                "passed": true,
                "detail": "assert prime_num(151) == True"
              },
              {
                "name": "Integer 152",
                "passed": true,
                "detail": "assert prime_num(152) == False"
              },
              {
                "name": "Integer 153",
                "passed": false,
                "detail": "assert prime_num(153) == False -> AssertionError"
              },
              {
                "name": "Integer 154",
                "passed": true,
                "detail": "assert prime_num(154) == False"
              },
              {
                "name": "Integer 155",
                "passed": false,
                "detail": "assert prime_num(155) == False -> AssertionError"
              },
              {
                "name": "Integer 156",
                "passed": true,
                "detail": "assert prime_num(156) == False"
              },
              {
                "name": "Integer 157",
                "passed": true,
                "detail": "assert prime_num(157) == True"
              },
              {
                "name": "Integer 158",
                "passed": true,
                "detail": "assert prime_num(158) == False"
              },
              {
                "name": "Integer 159",
                "passed": false,
                "detail": "assert prime_num(159) == False -> AssertionError"
              },
              {
                "name": "Integer 160",
                "passed": true,
                "detail": "assert prime_num(160) == False"
              },
              {
                "name": "Integer 161",
                "passed": false,
                "detail": "assert prime_num(161) == False -> AssertionError"
              },
              {
                "name": "Integer 162",
                "passed": true,
                "detail": "assert prime_num(162) == False"
              },
              {
                "name": "Integer 163",
                "passed": true,
                "detail": "assert prime_num(163) == True"
              },
              {
                "name": "Integer 164",
                "passed": true,
                "detail": "assert prime_num(164) == False"
              },
              {
                "name": "Integer 165",
                "passed": false,
                "detail": "assert prime_num(165) == False -> AssertionError"
              },
              {
                "name": "Integer 166",
                "passed": true,
                "detail": "assert prime_num(166) == False"
              },
              {
                "name": "Integer 167",
                "passed": true,
                "detail": "assert prime_num(167) == True"
              },
              {
                "name": "Integer 168",
                "passed": true,
                "detail": "assert prime_num(168) == False"
              },
              {
                "name": "Integer 169",
                "passed": false,
                "detail": "assert prime_num(169) == False -> AssertionError"
              },
              {
                "name": "Integer 170",
                "passed": true,
                "detail": "assert prime_num(170) == False"
              },
              {
                "name": "Integer 171",
                "passed": false,
                "detail": "assert prime_num(171) == False -> AssertionError"
              },
              {
                "name": "Integer 172",
                "passed": true,
                "detail": "assert prime_num(172) == False"
              },
              {
                "name": "Integer 173",
                "passed": true,
                "detail": "assert prime_num(173) == True"
              },
              {
                "name": "Integer 174",
                "passed": true,
                "detail": "assert prime_num(174) == False"
              },
              {
                "name": "Integer 175",
                "passed": false,
                "detail": "assert prime_num(175) == False -> AssertionError"
              },
              {
                "name": "Integer 176",
                "passed": true,
                "detail": "assert prime_num(176) == False"
              },
              {
                "name": "Integer 177",
                "passed": false,
                "detail": "assert prime_num(177) == False -> AssertionError"
              },
              {
                "name": "Integer 178",
                "passed": true,
                "detail": "assert prime_num(178) == False"
              },
              {
                "name": "Integer 179",
                "passed": true,
                "detail": "assert prime_num(179) == True"
              },
              {
                "name": "Integer 180",
                "passed": true,
                "detail": "assert prime_num(180) == False"
              },
              {
                "name": "Integer 181",
                "passed": true,
                "detail": "assert prime_num(181) == True"
              },
              {
                "name": "Integer 182",
                "passed": true,
                "detail": "assert prime_num(182) == False"
              },
              {
                "name": "Integer 183",
                "passed": false,
                "detail": "assert prime_num(183) == False -> AssertionError"
              },
              {
                "name": "Integer 184",
                "passed": true,
                "detail": "assert prime_num(184) == False"
              },
              {
                "name": "Integer 185",
                "passed": false,
                "detail": "assert prime_num(185) == False -> AssertionError"
              },
              {
                "name": "Integer 186",
                "passed": true,
                "detail": "assert prime_num(186) == False"
              },
              {
                "name": "Integer 187",
                "passed": false,
                "detail": "assert prime_num(187) == False -> AssertionError"
              },
              {
                "name": "Integer 188",
                "passed": true,
                "detail": "assert prime_num(188) == False"
              },
              {
                "name": "Integer 189",
                "passed": false,
                "detail": "assert prime_num(189) == False -> AssertionError"
              },
              {
                "name": "Integer 190",
                "passed": true,
                "detail": "assert prime_num(190) == False"
              },
              {
                "name": "Integer 191",
                "passed": true,
                "detail": "assert prime_num(191) == True"
              },
              {
                "name": "Integer 192",
                "passed": true,
                "detail": "assert prime_num(192) == False"
              },
              {
                "name": "Integer 193",
                "passed": true,
                "detail": "assert prime_num(193) == True"
              },
              {
                "name": "Integer 194",
                "passed": true,
                "detail": "assert prime_num(194) == False"
              },
              {
                "name": "Integer 195",
                "passed": false,
                "detail": "assert prime_num(195) == False -> AssertionError"
              },
              {
                "name": "Integer 196",
                "passed": true,
                "detail": "assert prime_num(196) == False"
              },
              {
                "name": "Integer 197",
                "passed": true,
                "detail": "assert prime_num(197) == True"
              },
              {
                "name": "Integer 198",
                "passed": true,
                "detail": "assert prime_num(198) == False"
              },
              {
                "name": "Integer 199",
                "passed": true,
                "detail": "assert prime_num(199) == True"
              },
              {
                "name": "Integer 200",
                "passed": true,
                "detail": "assert prime_num(200) == False"
              }
            ]
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Python subprocess execution against source assertions and enumerated supplemental contract cases.",
        "checks": [
          {
            "name": "Trial division through the square root",
            "passed": true,
            "detail": "Passed 3 source tests and 206 supplemental cases."
          },
          {
            "name": "Full divisor enumeration",
            "passed": true,
            "detail": "Passed 3 source tests and 206 supplemental cases."
          },
          {
            "name": "Incorrect control is rejected",
            "passed": true,
            "detail": "Control fails 59 of 209 cases. First failure: assert prime_num(1) == False -> AssertionError"
          },
          {
            "name": "Source reference audit",
            "passed": false,
            "detail": "150/209 cases pass. The dataset's own reference passes all three source tests, but returns None for 2 and True for 9. The reference is evidence to audit, not an infallible oracle."
          }
        ],
        "limitations": [
          "No separate model rubric judge was run.",
          "No performance requirement is imposed because the task does not request one.",
          "Ground-truth tests and source code can contain defects; the prime example deliberately retains a failed source reference."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:684397d188c1deceb49af332",
      "title": "Speculative episode outline: creative freedom",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "684397d188c1deceb49af332",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "a566ccecfd482e0c01f5967cf8cb0603896af016c13f9d3864eb792abdee7da4",
        "source_row_index": 78,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_provisional",
        "reason": "The primary deliverable is an original narrative outline with casting and filming suggestions."
      },
      "language": "en",
      "prompt": "Write an outline of a potential new episode for the Netflix show Black Mirror. This outline should be robust: for example, it should include descriptions of where the episode should be filmed and suggestions for potential actors for the roles.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response clearly states an episode title which is four words or less (e.g. \"In the Chamber\", \"Lost Taste\", \"Lake Silver\", \"Trail of Truth\").\n\n**ID:** c1\n\n**Criterion:** The response clearly states an episode title which is four words or less (e.g. \"In the Chamber\", \"Lost Taste\", \"Lake Silver\", \"Trail of Truth\").\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses and cites credible sources to create a story grounded in science (e.g. Article and research papers about robotics, cybernatics, public health, or artificial intelligence).\n\n**ID:** c2\n\n**Criterion:** The response uses and cites credible sources to create a story grounded in science (e.g. Article and research papers about robotics, cybernatics, public health, or artificial intelligence).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response is clearly presented in outline form (i.e. Short descriptions of the different scenes involved and other descriptions such as characters and themes clearly labelled as different sections).\n\n**ID:** c3\n\n**Criterion:** The response is clearly presented in outline form (i.e. Short descriptions of the different scenes involved and other descriptions such as characters and themes clearly labelled as different sections).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains narrative making up a significant portion of the outline's detail (ie., over 1/3 of the words are the narrative).\n\n**ID:** c4\n\n**Criterion:** The response contains narrative making up a significant portion of the outline's detail (ie., over 1/3 of the words are the narrative).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response divides the plot into at least three chronological segments (e.g., Act I/II/III or Beginning/Middle/End).\n\n**ID:** c5\n\n**Criterion:** The response divides the plot into at least three chronological segments (e.g., Act I/II/III or Beginning/Middle/End).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response specifies at least one real-world specific filming location (e.g., Los Angeles, New Zealand, Atlanta, London) and justifies the selections\n\n**ID:** c6\n\n**Criterion:** The response specifies at least one real-world specific filming location (e.g., Los Angeles, New Zealand, Atlanta, London) and justifies the selections\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response outlines a living actor (e.g, Anjana Vasan, Daniel Lapaine, Rashida Jones, David Tennant) to each character and justifies the selections.\n\n**ID:** c7\n\n**Criterion:** The response outlines a living actor (e.g, Anjana Vasan, Daniel Lapaine, Rashida Jones, David Tennant) to each character and justifies the selections.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response suggests more than two  A-List actors (e.g., Brad Pitt, Zendaya, Leonard DiCaprio, Meryl Streep)\n\n**ID:** c8\n\n**Criterion:** The response suggests more than two  A-List actors (e.g., Brad Pitt, Zendaya, Leonard DiCaprio, Meryl Streep)\n\n**Points:** -2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains at least one explicit reference (not just inspiration) to a previous black mirror episode (e.g., includes the song Anyone Who Knows What Love Is, United Kingdom News, Barnies, Streamberry).\n\n**ID:** c9\n\n**Criterion:** The response contains at least one explicit reference (not just inspiration) to a previous black mirror episode (e.g., includes the song Anyone Who Knows What Love Is, United Kingdom News, Barnies, Streamberry).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes any content which is not related to an episode outline (e.g., commentary on previous episodes, praise for the episode, discussion of technology, discussions of other series).\n\n**ID:** c10\n\n**Criterion:** The response includes any content which is not related to an episode outline (e.g., commentary on previous episodes, praise for the episode, discussion of technology, discussions of other series).\n\n**Points:** -3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes at least one plot twist.\n\n**ID:** c11\n\n**Criterion:** The response includes at least one plot twist.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes more than one plot twist.  \n\n**ID:** c12\n\n**Criterion:** The response includes more than one plot twist.  \n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response’s ending prompts discussion or thought (i.e., there is at least one lingering unresolved question).\n\n**ID:** c13\n\n**Criterion:** The response’s ending prompts discussion or thought (i.e., there is at least one lingering unresolved question).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response relies heavily on ideas or features from previous Black Mirror episodes – it is not sufficiently original.\n\n**ID:** c14\n\n**Criterion:** The response relies heavily on ideas or features from previous Black Mirror episodes – it is not sufficiently original.\n\n**Points:** -3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response’s contents are of an appropriate length: i.e., they would fit into an episode of 45-90 minutes.\n\n**ID:** c15\n\n**Criterion:** The response’s contents are of an appropriate length: i.e., they would fit into an episode of 45-90 minutes.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a potential composer (e.g., Bear McCreary, Ariel Marx, Brian Reitzell, Ames Bessada)\n\n**ID:** c16\n\n**Criterion:** The response includes a potential composer (e.g., Bear McCreary, Ariel Marx, Brian Reitzell, Ames Bessada)\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes relevant themes that the narrative resolves around (e.g., privacy, the internet, healthcare, recycling).\n\n**ID:** c17\n\n**Criterion:** The response includes relevant themes that the narrative resolves around (e.g., privacy, the internet, healthcare, recycling).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes an element which is clearly not feasible to produce (e.g., renting out time’s square to film for a week, requiring millions of detailed CGI objects, illegal content, copyright infringement).\n\n**ID:** c18\n\n**Criterion:** The response includes an element which is clearly not feasible to produce (e.g., renting out time’s square to film for a week, requiring millions of detailed CGI objects, illegal content, copyright infringement).\n\n**Points:** -2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response’s opening scene is normal/mundane/believable as reality (e.g., the scene could be in any other Netflix show, the scene does not have any social dynamics not present in 2025 US / UK society, the scene does not have any new technology, the scene does not involve the main theme in any way).\n\n**ID:** c19\n\n**Criterion:** The response’s opening scene is normal/mundane/believable as reality (e.g., the scene could be in any other Netflix show, the scene does not have any social dynamics not present in 2025 US / UK society, the scene does not have any new technology, the scene does not involve the main theme in any way).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses technicians and experts to involve in production (e.g., Russell McLean, Digital Orchard, James MacLachlan, Ingenuity Studios).\n\n**ID:** c20\n\n**Criterion:** The response discusses technicians and experts to involve in production (e.g., Russell McLean, Digital Orchard, James MacLachlan, Ingenuity Studios).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes budget propositoning and justifications (e.g., the budget is at least $1 million, the salaries for potential actors are proposed, the special effects' costs are outlined, the filming costs are mentioned).\n\n**ID:** c21\n\n**Criterion:** The response includes budget propositoning and justifications (e.g., the budget is at least $1 million, the salaries for potential actors are proposed, the special effects' costs are outlined, the filming costs are mentioned).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes one topic relevant for 2026 (e.g., war, authoritarionism, youth unemployment, immigration).\n\n**ID:** c22\n\n**Criterion:** The response includes one topic relevant for 2026 (e.g., war, authoritarionism, youth unemployment, immigration).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response's plot is exclusively positive.\n\n**ID:** c23\n\n**Criterion:** The response's plot is exclusively positive.\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response clearly states an episode title which is four words or less (e.g. \"In the Chamber\", \"Lost Taste\", \"Lake Silver\", \"Trail of Truth\").",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response uses and cites credible sources to create a story grounded in science (e.g. Article and research papers about robotics, cybernatics, public health, or artificial intelligence).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response is clearly presented in outline form (i.e. Short descriptions of the different scenes involved and other descriptions such as characters and themes clearly labelled as different sections).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response contains narrative making up a significant portion of the outline's detail (ie., over 1/3 of the words are the narrative).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response divides the plot into at least three chronological segments (e.g., Act I/II/III or Beginning/Middle/End).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response specifies at least one real-world specific filming location (e.g., Los Angeles, New Zealand, Atlanta, London) and justifies the selections",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response outlines a living actor (e.g, Anjana Vasan, Daniel Lapaine, Rashida Jones, David Tennant) to each character and justifies the selections.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response suggests more than two  A-List actors (e.g., Brad Pitt, Zendaya, Leonard DiCaprio, Meryl Streep)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": -2.0
            },
            {
              "criterion": "The response contains at least one explicit reference (not just inspiration) to a previous black mirror episode (e.g., includes the song Anyone Who Knows What Love Is, United Kingdom News, Barnies, Streamberry).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes any content which is not related to an episode outline (e.g., commentary on previous episodes, praise for the episode, discussion of technology, discussions of other series).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": -3.0
            },
            {
              "criterion": "The response includes at least one plot twist.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes more than one plot twist.  ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response’s ending prompts discussion or thought (i.e., there is at least one lingering unresolved question).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response relies heavily on ideas or features from previous Black Mirror episodes – it is not sufficiently original.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": -3.0
            },
            {
              "criterion": "The response’s contents are of an appropriate length: i.e., they would fit into an episode of 45-90 minutes.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response includes a potential composer (e.g., Bear McCreary, Ariel Marx, Brian Reitzell, Ames Bessada)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes relevant themes that the narrative resolves around (e.g., privacy, the internet, healthcare, recycling).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes an element which is clearly not feasible to produce (e.g., renting out time’s square to film for a week, requiring millions of detailed CGI objects, illegal content, copyright infringement).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": -2.0
            },
            {
              "criterion": "The response’s opening scene is normal/mundane/believable as reality (e.g., the scene could be in any other Netflix show, the scene does not have any social dynamics not present in 2025 US / UK society, the scene does not have any new technology, the scene does not involve the main theme in any way).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response discusses technicians and experts to involve in production (e.g., Russell McLean, Digital Orchard, James MacLachlan, Ingenuity Studios).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes budget propositoning and justifications (e.g., the budget is at least $1 million, the salaries for potential actors are proposed, the special effects' costs are outlined, the filming costs are mentioned).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes one topic relevant for 2026 (e.g., war, authoritarionism, youth unemployment, immigration).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c22",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response's plot is exclusively positive.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c23",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 23,
        "source_original_items": [
          {
            "axis": "Implicit Criteria",
            "criterion": "The response clearly states an episode title which is four words or less (e.g. \"In the Chamber\", \"Lost Taste\", \"Lake Silver\", \"Trail of Truth\").",
            "weight": 2.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response uses and cites credible sources to create a story grounded in science (e.g. Article and research papers about robotics, cybernatics, public health, or artificial intelligence).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response is clearly presented in outline form (i.e. Short descriptions of the different scenes involved and other descriptions such as characters and themes clearly labelled as different sections).",
            "weight": 5.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response contains narrative making up a significant portion of the outline's detail (ie., over 1/3 of the words are the narrative).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response divides the plot into at least three chronological segments (e.g., Act I/II/III or Beginning/Middle/End).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response specifies at least one real-world specific filming location (e.g., Los Angeles, New Zealand, Atlanta, London) and justifies the selections",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response outlines a living actor (e.g, Anjana Vasan, Daniel Lapaine, Rashida Jones, David Tennant) to each character and justifies the selections.",
            "weight": 5.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response suggests more than two  A-List actors (e.g., Brad Pitt, Zendaya, Leonard DiCaprio, Meryl Streep)",
            "weight": -2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response contains at least one explicit reference (not just inspiration) to a previous black mirror episode (e.g., includes the song Anyone Who Knows What Love Is, United Kingdom News, Barnies, Streamberry).",
            "weight": 4.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes any content which is not related to an episode outline (e.g., commentary on previous episodes, praise for the episode, discussion of technology, discussions of other series).",
            "weight": -3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes at least one plot twist.",
            "weight": 4.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes more than one plot twist.  ",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response’s ending prompts discussion or thought (i.e., there is at least one lingering unresolved question).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response relies heavily on ideas or features from previous Black Mirror episodes – it is not sufficiently original.",
            "weight": -3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response’s contents are of an appropriate length: i.e., they would fit into an episode of 45-90 minutes.",
            "weight": 5.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes a potential composer (e.g., Bear McCreary, Ariel Marx, Brian Reitzell, Ames Bessada)",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes relevant themes that the narrative resolves around (e.g., privacy, the internet, healthcare, recycling).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes an element which is clearly not feasible to produce (e.g., renting out time’s square to film for a week, requiring millions of detailed CGI objects, illegal content, copyright infringement).",
            "weight": -2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response’s opening scene is normal/mundane/believable as reality (e.g., the scene could be in any other Netflix show, the scene does not have any social dynamics not present in 2025 US / UK society, the scene does not have any new technology, the scene does not involve the main theme in any way).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response discusses technicians and experts to involve in production (e.g., Russell McLean, Digital Orchard, James MacLachlan, Ingenuity Studios).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes budget propositoning and justifications (e.g., the budget is at least $1 million, the salaries for potential actors are proposed, the special effects' costs are outlined, the filming costs are mentioned).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes one topic relevant for 2026 (e.g., war, authoritarionism, youth unemployment, immigration).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response's plot is exclusively positive.",
            "weight": -5.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Rewards a producible plot, locations, role casting and narrative development.",
          "Explicitly checks originality instead of accepting copied premises."
        ],
        "risks": [
          "Source demands ≤4-word title, multiple twists, composer, budget, 2026 theme and a mundane opening; most are not prompt requirements.",
          "A -5 penalty for an exclusively positive plot enforces a particular ending tone.",
          "Actor availability and budget feasibility have not been researched for the sketches."
        ],
        "must_hold": [
          "A robust original episode outline, filming locations and casting suggestions.",
          "Coherent cause and effect, credible motivations and a technically or socially meaningful speculative premise."
        ],
        "may_vary": [
          "One twist versus several, chronology, title length, ending tone, actors and locations.",
          "Different social questions and technologies, provided the result answers the creative brief."
        ],
        "proposed_changes": [
          "Score speculative tension and narrative coherence without prescribing a specific ending or opening.",
          "Treat budget, composer and continuity references as optional unless requested.",
          "Use two independently invented plots to check that anchors are examples, not required story beats."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: The Waiting Room",
          "kind": "valid_alternative",
          "response": "A municipal queue app lets residents sell waiting time. A hospital cleaner uses it to help a neighbor, discovers urgent cases are being quietly deprioritized, and must decide whether publishing the exploit harms those still waiting. A three-act outline could use Glasgow municipal interiors and suggest Gugu Mbatha-Raw for the conflicted protagonist, with casting and location feasibility still to verify.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Different premise and a morally unresolved ending; this is not a complete production outline."
        },
        {
          "label": "Sketch B: Archive of Small Mercies",
          "kind": "valid_alternative",
          "response": "A neighborhood device lets people temporarily experience another resident's overlooked acts of care. An attempted political manipulation fails when users collectively reveal the mechanism; the hopeful ending asks whether empathy can survive after the device is removed. An outline could use Bristol streets and suggest Wunmi Mosaku as the organizer, with feasibility still to verify.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A hopeful resolution can still explore power and technology; the source's blanket positive-ending penalty may reject it."
        },
        {
          "label": "Invalid: omit the requested outline",
          "kind": "invalid_control",
          "response": "Film somewhere nice and hire a famous actor. Technology is bad.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Provides neither a robust plot nor concrete filming/casting suggestions."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605351",
      "title": "SVM versus logistic regression: many good explanations",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605351",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "0ac0a684a93cc2d585a949c8ef68fb53cd27e77dc0a674580714199da6df8b0e",
        "source_row_index": 31,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_provisional",
        "reason": "The prompt asks for an explanatory article, not an implemented model or a statistical estimate."
      },
      "language": "en",
      "prompt": "Write an explanatory article comparing and contrasting Support Vector Machine and Logistic Regression.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response contains at least four explicit sentences or bullets describing similarities shared by Support Vector Machine (SVM) and Logistic Regression (e.g. both are supervised learning algorithms for classification, both are linear classifiers, can handle non-linear data, uses regularization).\n\n**ID:** c1\n\n**Criterion:** The response contains at least four explicit sentences or bullets describing similarities shared by Support Vector Machine (SVM) and Logistic Regression (e.g. both are supervised learning algorithms for classification, both are linear classifiers, can handle non-linear data, uses regularization).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains at least four explicit sentences or bullets describing differences between Support Vector Machine (SVM) and Logistic Regression (e.g. SVM maximizes margin while Logistic Regression maximizes likelihood, SVM is geometric while Logistic Regression is statistical, SVM is robust to outliers while Logistic Regression is more sensitive).\n\n**ID:** c2\n\n**Criterion:** The response contains at least four explicit sentences or bullets describing differences between Support Vector Machine (SVM) and Logistic Regression (e.g. SVM maximizes margin while Logistic Regression maximizes likelihood, SVM is geometric while Logistic Regression is statistical, SVM is robust to outliers while Logistic Regression is more sensitive).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes executable code snippets as examples of how to use Support Vector Machine or Logistic Regression (e.g. setup codes, library imports, example usage).\n\n**ID:** c3\n\n**Criterion:** The response includes executable code snippets as examples of how to use Support Vector Machine or Logistic Regression (e.g. setup codes, library imports, example usage).\n\n**Points:** -2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly states that Support Vector Machine constructs a decision boundary or hyperplane that maximizes the margin between classes, and how support vectors define the SVM model.\n\n**ID:** c4\n\n**Criterion:** The response correctly states that Support Vector Machine constructs a decision boundary or hyperplane that maximizes the margin between classes, and how support vectors define the SVM model.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly states that Logistic Regression models class probability with the logistic sigmoid function or log-odds formulation.\n\n**ID:** c5\n\n**Criterion:** The response correctly states that Logistic Regression models class probability with the logistic sigmoid function or log-odds formulation.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly states that Support Vector Machines can handle non-linear relationships by using kernel functions (e.g. linear kernel, Radial Basis Function(RBF), polynomial, sigmoid).\n\n**ID:** c6\n\n**Criterion:** The response correctly states that Support Vector Machines can handle non-linear relationships by using kernel functions (e.g. linear kernel, Radial Basis Function(RBF), polynomial, sigmoid).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly explains the difference between hard-margin and soft-margin Support Vector Machines (e.g. hard-margin SVM requires perfect separation with no errors; soft-margin SVM allows some misclassifications to improve generalization).\n\n**ID:** c7\n\n**Criterion:** The response correctly explains the difference between hard-margin and soft-margin Support Vector Machines (e.g. hard-margin SVM requires perfect separation with no errors; soft-margin SVM allows some misclassifications to improve generalization).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly states that the standard Logistic Regression assumes a linear decision boundary in the feature space unless features are transformed.\n\n**ID:** c8\n\n**Criterion:** The response correctly states that the standard Logistic Regression assumes a linear decision boundary in the feature space unless features are transformed.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response correctly states that Support Vector Machine produces label predictions, not probabilities by default.\n\n**ID:** c9\n\n**Criterion:** The response correctly states that Support Vector Machine produces label predictions, not probabilities by default.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains the derivation of the objective function for Support Vector Machine, which includes hinge loss and regularization parameters.\n\n**ID:** c10\n\n**Criterion:** The response explains the derivation of the objective function for Support Vector Machine, which includes hinge loss and regularization parameters.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains the derivation of the Logistic Regression from Gaussian Naive Bayes.\n\n**ID:** c11\n\n**Criterion:** The response explains the derivation of the Logistic Regression from Gaussian Naive Bayes.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions that Support Vector Machine is a quadratic programming problem.\n\n**ID:** c12\n\n**Criterion:** The response mentions that Support Vector Machine is a quadratic programming problem.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains the formula for Support Vector Machine's dual formulation.\n\n**ID:** c13\n\n**Criterion:** The response contains the formula for Support Vector Machine's dual formulation.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses at least five distinct dimensions of comparison for Support Vector Machine and Logistic Regression (e.g. mathematical objective, decision boundary shape, interpretability, computational complexity, robustness to outliers, scalability, regularization, typical data‐set size/shape).\n\n**ID:** c14\n\n**Criterion:** The response discusses at least five distinct dimensions of comparison for Support Vector Machine and Logistic Regression (e.g. mathematical objective, decision boundary shape, interpretability, computational complexity, robustness to outliers, scalability, regularization, typical data‐set size/shape).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one practical scenario where Support Vector Machine is preferable, justified by its attributes (e.g. effective in high-dimensional spaces, robust to overfitting/outlier, applicable to small dataset).\n\n**ID:** c15\n\n**Criterion:** The response provides at least one practical scenario where Support Vector Machine is preferable, justified by its attributes (e.g. effective in high-dimensional spaces, robust to overfitting/outlier, applicable to small dataset).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one practical scenario where Logistic Regression is preferable, justified by its attributes (e.g. simplicity, computationally efficient, effective for binary classification).\n\n**ID:** c16\n\n**Criterion:** The response provides at least one practical scenario where Logistic Regression is preferable, justified by its attributes (e.g. simplicity, computationally efficient, effective for binary classification).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes practical use-case scenarios for Support Vector Machine based on its properties (e.g. bioinformatics, handwriting recognition, face recognition).\n\n**ID:** c17\n\n**Criterion:** The response describes practical use-case scenarios for Support Vector Machine based on its properties (e.g. bioinformatics, handwriting recognition, face recognition).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes practical use-case scenarios for Logistic Regression based on its properties (e.g. fraud detection, spam detection, loan approval, disease prediction).\n\n**ID:** c18\n\n**Criterion:** The response describes practical use-case scenarios for Logistic Regression based on its properties (e.g. fraud detection, spam detection, loan approval, disease prediction).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains a summary of Support Vector Machine and Logistic Regression's comparison (e.g. synthesizing key similarities, differences, take-home messages).\n\n**ID:** c19\n\n**Criterion:** The response contains a summary of Support Vector Machine and Logistic Regression's comparison (e.g. synthesizing key similarities, differences, take-home messages).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response compares Support Vector Machine and Logistic Regression's convexity properties in optimization problems.\n\n**ID:** c20\n\n**Criterion:** The response compares Support Vector Machine and Logistic Regression's convexity properties in optimization problems.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains the geometric intuition of the margin in Support Vector Machine.\n\n**ID:** c21\n\n**Criterion:** The response explains the geometric intuition of the margin in Support Vector Machine.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses regularization techniques (e.g. the L1/L2 for Logistic Regression, C parameter in Support Vector Machine).\n\n**ID:** c22\n\n**Criterion:** The response discusses regularization techniques (e.g. the L1/L2 for Logistic Regression, C parameter in Support Vector Machine).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains how gradient descent or related optimization methods are used to optimize loss functions for both Support Vector Machine and Logistic Regression.\n\n**ID:** c23\n\n**Criterion:** The response explains how gradient descent or related optimization methods are used to optimize loss functions for both Support Vector Machine and Logistic Regression.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a chart or table that compares Support Vector Machine and Logistic Regression across multiple dimensions in a condensed form (e.g. dimension, dataset size, computational efficiency, robustness to outliers).\n\n**ID:** c24\n\n**Criterion:** The response includes a chart or table that compares Support Vector Machine and Logistic Regression across multiple dimensions in a condensed form (e.g. dimension, dataset size, computational efficiency, robustness to outliers).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response incorporates a historical or theoretical context behind Logistic Regression and Support Vector Machine (e.g. origins of SVM in Vapnik-Chervonenkis theory, Logistic Regression in statistics).\n\n**ID:** c25\n\n**Criterion:** The response incorporates a historical or theoretical context behind Logistic Regression and Support Vector Machine (e.g. origins of SVM in Vapnik-Chervonenkis theory, Logistic Regression in statistics).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response compares model outputs in multi-class scenarios by introducing extensions like One-vs-All or Multinomial Logistic Regression.\n\n**ID:** c26\n\n**Criterion:** The response compares model outputs in multi-class scenarios by introducing extensions like One-vs-All or Multinomial Logistic Regression.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains how Support Vector Machine and Logistic Regression behave on imbalanced datasets (e.g. decision boundary skewing, loss function bias, probability threshold issues) and their mitigation techniques (e.g. class weights, sampling strategies).\n\n**ID:** c27\n\n**Criterion:** The response explains how Support Vector Machine and Logistic Regression behave on imbalanced datasets (e.g. decision boundary skewing, loss function bias, probability threshold issues) and their mitigation techniques (e.g. class weights, sampling strategies).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses the impact of feature scaling on the Support Vector Machine and Logistic Regression (e.g. distance metrics, hyperplane skewing, gradient descent convergence, regularization impact).\n\n**ID:** c28\n\n**Criterion:** The response discusses the impact of feature scaling on the Support Vector Machine and Logistic Regression (e.g. distance metrics, hyperplane skewing, gradient descent convergence, regularization impact).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Mathematical equations are displayed in clear mathematical notation like LaTex format.\n\n**ID:** c29\n\n**Criterion:** Mathematical equations are displayed in clear mathematical notation like LaTex format.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses other alternative algorithms for classification (e.g. decision trees, k-nearest neighbors).\n\n**ID:** c30\n\n**Criterion:** The response discusses other alternative algorithms for classification (e.g. decision trees, k-nearest neighbors).\n\n**Points:** -3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response contains at least four explicit sentences or bullets describing similarities shared by Support Vector Machine (SVM) and Logistic Regression (e.g. both are supervised learning algorithms for classification, both are linear classifiers, can handle non-linear data, uses regularization).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response contains at least four explicit sentences or bullets describing differences between Support Vector Machine (SVM) and Logistic Regression (e.g. SVM maximizes margin while Logistic Regression maximizes likelihood, SVM is geometric while Logistic Regression is statistical, SVM is robust to outliers while Logistic Regression is more sensitive).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response includes executable code snippets as examples of how to use Support Vector Machine or Logistic Regression (e.g. setup codes, library imports, example usage).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": -2.0
            },
            {
              "criterion": "The response correctly states that Support Vector Machine constructs a decision boundary or hyperplane that maximizes the margin between classes, and how support vectors define the SVM model.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response correctly states that Logistic Regression models class probability with the logistic sigmoid function or log-odds formulation.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response correctly states that Support Vector Machines can handle non-linear relationships by using kernel functions (e.g. linear kernel, Radial Basis Function(RBF), polynomial, sigmoid).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response correctly explains the difference between hard-margin and soft-margin Support Vector Machines (e.g. hard-margin SVM requires perfect separation with no errors; soft-margin SVM allows some misclassifications to improve generalization).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response correctly states that the standard Logistic Regression assumes a linear decision boundary in the feature space unless features are transformed.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response correctly states that Support Vector Machine produces label predictions, not probabilities by default.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response explains the derivation of the objective function for Support Vector Machine, which includes hinge loss and regularization parameters.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response explains the derivation of the Logistic Regression from Gaussian Naive Bayes.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response mentions that Support Vector Machine is a quadratic programming problem.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response contains the formula for Support Vector Machine's dual formulation.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response discusses at least five distinct dimensions of comparison for Support Vector Machine and Logistic Regression (e.g. mathematical objective, decision boundary shape, interpretability, computational complexity, robustness to outliers, scalability, regularization, typical data‐set size/shape).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response provides at least one practical scenario where Support Vector Machine is preferable, justified by its attributes (e.g. effective in high-dimensional spaces, robust to overfitting/outlier, applicable to small dataset).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response provides at least one practical scenario where Logistic Regression is preferable, justified by its attributes (e.g. simplicity, computationally efficient, effective for binary classification).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response describes practical use-case scenarios for Support Vector Machine based on its properties (e.g. bioinformatics, handwriting recognition, face recognition).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response describes practical use-case scenarios for Logistic Regression based on its properties (e.g. fraud detection, spam detection, loan approval, disease prediction).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response contains a summary of Support Vector Machine and Logistic Regression's comparison (e.g. synthesizing key similarities, differences, take-home messages).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response compares Support Vector Machine and Logistic Regression's convexity properties in optimization problems.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response explains the geometric intuition of the margin in Support Vector Machine.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response discusses regularization techniques (e.g. the L1/L2 for Logistic Regression, C parameter in Support Vector Machine).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c22",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response explains how gradient descent or related optimization methods are used to optimize loss functions for both Support Vector Machine and Logistic Regression.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c23",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes a chart or table that compares Support Vector Machine and Logistic Regression across multiple dimensions in a condensed form (e.g. dimension, dataset size, computational efficiency, robustness to outliers).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c24",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response incorporates a historical or theoretical context behind Logistic Regression and Support Vector Machine (e.g. origins of SVM in Vapnik-Chervonenkis theory, Logistic Regression in statistics).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c25",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response compares model outputs in multi-class scenarios by introducing extensions like One-vs-All or Multinomial Logistic Regression.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c26",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response explains how Support Vector Machine and Logistic Regression behave on imbalanced datasets (e.g. decision boundary skewing, loss function bias, probability threshold issues) and their mitigation techniques (e.g. class weights, sampling strategies).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c27",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response discusses the impact of feature scaling on the Support Vector Machine and Logistic Regression (e.g. distance metrics, hyperplane skewing, gradient descent convergence, regularization impact).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c28",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Mathematical equations are displayed in clear mathematical notation like LaTex format.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c29",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response discusses other alternative algorithms for classification (e.g. decision trees, k-nearest neighbors).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c30",
              "method": "llm",
              "status": "provisional",
              "weight": -3.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 30,
        "source_original_items": [
          {
            "axis": "Synthesis of Information",
            "criterion": "The response contains at least four explicit sentences or bullets describing similarities shared by Support Vector Machine (SVM) and Logistic Regression (e.g. both are supervised learning algorithms for classification, both are linear classifiers, can handle non-linear data, uses regularization).",
            "weight": 4.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response contains at least four explicit sentences or bullets describing differences between Support Vector Machine (SVM) and Logistic Regression (e.g. SVM maximizes margin while Logistic Regression maximizes likelihood, SVM is geometric while Logistic Regression is statistical, SVM is robust to outliers while Logistic Regression is more sensitive).",
            "weight": 5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes executable code snippets as examples of how to use Support Vector Machine or Logistic Regression (e.g. setup codes, library imports, example usage).",
            "weight": -2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response correctly states that Support Vector Machine constructs a decision boundary or hyperplane that maximizes the margin between classes, and how support vectors define the SVM model.",
            "weight": 4.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response correctly states that Logistic Regression models class probability with the logistic sigmoid function or log-odds formulation.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response correctly states that Support Vector Machines can handle non-linear relationships by using kernel functions (e.g. linear kernel, Radial Basis Function(RBF), polynomial, sigmoid).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response correctly explains the difference between hard-margin and soft-margin Support Vector Machines (e.g. hard-margin SVM requires perfect separation with no errors; soft-margin SVM allows some misclassifications to improve generalization).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response correctly states that the standard Logistic Regression assumes a linear decision boundary in the feature space unless features are transformed.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response correctly states that Support Vector Machine produces label predictions, not probabilities by default.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains the derivation of the objective function for Support Vector Machine, which includes hinge loss and regularization parameters.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains the derivation of the Logistic Regression from Gaussian Naive Bayes.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response mentions that Support Vector Machine is a quadratic programming problem.",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response contains the formula for Support Vector Machine's dual formulation.",
            "weight": 1.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response discusses at least five distinct dimensions of comparison for Support Vector Machine and Logistic Regression (e.g. mathematical objective, decision boundary shape, interpretability, computational complexity, robustness to outliers, scalability, regularization, typical data‐set size/shape).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response provides at least one practical scenario where Support Vector Machine is preferable, justified by its attributes (e.g. effective in high-dimensional spaces, robust to overfitting/outlier, applicable to small dataset).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response provides at least one practical scenario where Logistic Regression is preferable, justified by its attributes (e.g. simplicity, computationally efficient, effective for binary classification).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response describes practical use-case scenarios for Support Vector Machine based on its properties (e.g. bioinformatics, handwriting recognition, face recognition).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response describes practical use-case scenarios for Logistic Regression based on its properties (e.g. fraud detection, spam detection, loan approval, disease prediction).",
            "weight": 1.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response contains a summary of Support Vector Machine and Logistic Regression's comparison (e.g. synthesizing key similarities, differences, take-home messages).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response compares Support Vector Machine and Logistic Regression's convexity properties in optimization problems.",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains the geometric intuition of the margin in Support Vector Machine.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses regularization techniques (e.g. the L1/L2 for Logistic Regression, C parameter in Support Vector Machine).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains how gradient descent or related optimization methods are used to optimize loss functions for both Support Vector Machine and Logistic Regression.",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes a chart or table that compares Support Vector Machine and Logistic Regression across multiple dimensions in a condensed form (e.g. dimension, dataset size, computational efficiency, robustness to outliers).",
            "weight": 3.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response incorporates a historical or theoretical context behind Logistic Regression and Support Vector Machine (e.g. origins of SVM in Vapnik-Chervonenkis theory, Logistic Regression in statistics).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response compares model outputs in multi-class scenarios by introducing extensions like One-vs-All or Multinomial Logistic Regression.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains how Support Vector Machine and Logistic Regression behave on imbalanced datasets (e.g. decision boundary skewing, loss function bias, probability threshold issues) and their mitigation techniques (e.g. class weights, sampling strategies).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses the impact of feature scaling on the Support Vector Machine and Logistic Regression (e.g. distance metrics, hyperplane skewing, gradient descent convergence, regularization impact).",
            "weight": 1.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "Mathematical equations are displayed in clear mathematical notation like LaTex format.",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response discusses other alternative algorithms for classification (e.g. decision trees, k-nearest neighbors).",
            "weight": -3.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Covers objectives, outputs, scaling, class imbalance and practical selection.",
          "Signed penalties make restrictive criteria visible for inspection."
        ],
        "risks": [
          "c11 requires a Gaussian Naive Bayes derivation although the prompt asks only for a comparison.",
          "c1/c2 count sentences; c24 mandates a table; c3 penalizes any executable example. These can reject good explanations.",
          "Examples suggesting SVM is inherently more robust to outliers need qualification; this is not a universal ordering."
        ],
        "must_hold": [
          "Correct margin-versus-likelihood distinction and probability-versus-score distinction.",
          "Explain meaningful similarities and differences without universal performance claims."
        ],
        "may_vary": [
          "Intuitive versus mathematical depth; table versus connected prose; examples and sequence.",
          "A correct optional code illustration, unless the user specifies a no-code format."
        ],
        "proposed_changes": [
          "Replace sentence counts and mandatory table with coverage and clarity criteria.",
          "Make dual/Gaussian-Naive-Bayes derivations optional enrichment; condition any generative derivation on its assumptions.",
          "Allow equivalent loss conventions and accurate small code examples; remove generic outlier-superiority assumptions."
        ]
      },
      "probes": [
        {
          "label": "Candidate A: equation and comparison table",
          "kind": "valid_alternative",
          "response": "# Two ways to draw a classification boundary\n\nSupport vector machines (SVMs) and logistic regression both learn from labeled examples. Their standard linear versions separate classes with a hyperplane. Both can use regularization to control model complexity, and both benefit from consistent feature scaling. Neither method automatically solves class imbalance: class weights, suitable evaluation metrics, and—in probability-based decisions—threshold selection remain important.\n\nLogistic regression models a probability. For features x, it uses p(y=1|x)=1/(1+exp(-(w·x+b))). Thus its log-odds are linear in the features. Training minimizes negative log-likelihood, often with L1 or L2 regularization. For fixed features this is convex; gradient-based methods are common. Coefficients describe changes in log-odds conditional on the other features, though correlated features can complicate interpretation. The probabilities should still be checked for calibration on representative data.\n\nAn SVM instead seeks a large margin: a wide separation between the boundary and the closest constraining observations, called support vectors. A hard-margin SVM requires separable training data. A soft-margin SVM trades off margin width against hinge-loss violations; in one convention its objective is ½||w||² + C Σ max(0, 1-y_i(w·x_i+b)). Larger C puts more emphasis on training violations. A linear SVM is a convex optimization problem; its hinge loss can be handled by subgradient or specialized solvers. SVM decision scores are not probabilities unless an additional calibration procedure is fitted.\n\n| Dimension | Logistic regression | SVM |\n|---|---|---|\n| Objective | Conditional likelihood / log loss | Margin / hinge loss |\n| Native output | Estimated class probability | Decision score and class |\n| Nonlinearity | Transform the input features | Transform features or use kernels |\n| Interpretation | Feature effects on log-odds | Boundary and support vectors |\n| Scaling | Linear variants suit large sparse datasets | Linear variants scale well; kernel variants can be expensive |\n\nAn RBF or polynomial kernel lets an SVM represent a nonlinear boundary without explicitly constructing every transformed feature. Ordinary logistic regression is linear in its chosen feature space, so nonlinear behavior requires feature engineering or a richer feature mapping. This does not mean SVMs always outperform logistic regression or always resist outliers: loss, regularization, scale, and data contamination matter for both.\n\nFor a modest-sized dataset of nonlinear measurements, a cross-validated kernel SVM can be attractive. For a large sparse classification task where probabilistic decisions and coefficient interpretation matter, regularized logistic regression is a useful baseline. For multiple classes, SVMs can combine binary classifiers; logistic regression can use a multinomial softmax model. Choose using held-out performance, calibration when relevant, and operational cost—not the algorithm's name.",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A complete explanation with equations/table; intentionally does not earn every optional-detail point of the source rubric."
        },
        {
          "label": "Candidate B: conceptual prose",
          "kind": "valid_alternative",
          "response": "# Classification: probabilities or margins?\n\nImagine separating labeled messages into two classes. Logistic regression asks which linear score best predicts the labels probabilistically. Its sigmoid converts the score into a number between zero and one; fitting minimizes log loss, possibly with an L1 or L2 penalty. SVMs ask for a boundary with a large geometric margin while penalizing examples that fall within that margin or on its wrong side. Points constraining that boundary are support vectors.\n\nBoth are supervised classifiers. Both have linear versions. Both can regularize their fitted coefficients. Both can handle nonlinear structure if given suitable transformed features; an SVM additionally supports a kernel-based representation. Both need careful preprocessing, representative validation, and attention to class imbalance.\n\nThe main differences are the training objective, the native output, the nonlinear representation, and the computational profile. Logistic regression's objective comes from conditional likelihood; an SVM's comes from margin optimization. Logistic regression produces estimated probabilities, whereas an SVM normally produces signed scores and labels. SVM kernels can provide nonlinear boundaries; standard logistic regression remains linear in the supplied feature space. A kernel SVM may need many pairwise similarities and support vectors, while a linear model can be economical for a very large sparse dataset.\n\nA hard-margin SVM permits no classification violations and therefore requires linear separability in its feature space. A soft-margin SVM allows violations, controlled jointly by its hinge loss and regularization. With fixed features and the usual convex penalties, both standard logistic regression and linear SVM fitting are convex; smooth solvers suit logistic loss, whereas hinge loss calls for a solver that handles its nondifferentiable points. Larger feature scales can distort the regularization tradeoff, so fit scaling on training data and apply it consistently.\n\nIf nonlinear structure is important and the dataset is moderate, try a tuned kernel SVM. If you need a scalable probabilistic baseline or interpretable feature effects, try logistic regression. Logistic coefficients indicate effects on log-odds rather than causal effects, and probability quality must be evaluated separately from classification accuracy. For imbalanced labels, consider class weights and precision/recall-sensitive evaluation rather than accuracy alone. Multinomial logistic regression and multiclass combinations of SVMs extend the binary setting. Neither method is universally more robust to outliers: that depends on the loss, penalty, feature scaling, and the data.\n\nThe choice is therefore empirical: compare appropriate variants on held-out data and choose the boundary representation and output that fit the application.",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Explains the same central facts without a table or Naive Bayes derivation, exposing unrequested form/depth constraints."
        },
        {
          "label": "Invalid: swapped objectives and outputs",
          "kind": "invalid_control",
          "response": "SVM learns calibrated probabilities by maximum likelihood. Logistic regression maximizes the geometric margin and cannot output probabilities.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Reverses core mechanisms, so format flexibility must not rescue it."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605433",
      "title": "AI startup brainstorm: distinct ideas without one template",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605433",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "dd40cd6e476499a4bbd1cc1a0e5cab0b0536a62ff6c2164fe8b78cedc94c566c",
        "source_row_index": 1,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_provisional",
        "reason": "The deliverable is a three-idea technical brainstorm, not a built system or an operational company."
      },
      "language": "en",
      "prompt": "Act as a creative technical founder and brainstorm an innovative AI startup idea that meets these criteria:\n1. Leverages AI in a meaningful, technically interesting way, not just a thin wrapper around an LLM\n2. Uses or integrates APIs (e.g. for data sources, automation, or integration with popular tools)\n3. Solves a real, cool problem in people’s lives or work, something practical but original\n4. Has potential for viral growth or clear product-market fit\n\nContext: I like startups like Cal.ai (http://cal.ai) (AI calorie counting), Boardy.ai (http://boardy.ai) (AI networking), and AI scheduling assistants, things that automate or enhance daily life or work through smart AI tooling. \n\nPlease:\n* Suggest 3 ideas, with a clear description of the product and how it works technically (tech stack, APIs, AI models)\n* Explain what problem it solves and why people would want it\n* Keep the ideas ambitious but buildable, something that could actually become a startup\n\nFor each idea state if any company is doing something similar already (who are the competitors). I want the startup to leverage a technical advantage of using cool ai features only enabled now and that will improve as foundational models and services get better.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response provides exactly 3 startup ideas as requested.\n\n**ID:** c1\n\n**Criterion:** The response provides exactly 3 startup ideas as requested.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes ideas that can be built using modern available tools (e.g., AWS, Hugging Face, GPUs, LLMs).\n\n**ID:** c2\n\n**Criterion:** The response includes ideas that can be built using modern available tools (e.g., AWS, Hugging Face, GPUs, LLMs).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes how the ideas solve a real, practical problem (e.g., by identifying target users, addressing problems that people can plausibly face, tackling problems that require high compute).\n\n**ID:** c3\n\n**Criterion:** The response describes how the ideas solve a real, practical problem (e.g., by identifying target users, addressing problems that people can plausibly face, tackling problems that require high compute).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response substantiates its core claims with relevant, credible, and up-to-date authoritative sources (e.g., respected magazines like Forbes, credible VC reports / blog posts, economic reports from reputed think thanks, government reports), demonstrating a strong evidence-based foundation for its conclusions.\n\n**ID:** c4\n\n**Criterion:** The response substantiates its core claims with relevant, credible, and up-to-date authoritative sources (e.g., respected magazines like Forbes, credible VC reports / blog posts, economic reports from reputed think thanks, government reports), demonstrating a strong evidence-based foundation for its conclusions.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response states the technical advantages enabled by today’s AI advances for each proposed idea (e.g., having pretraining knowledge, in-context learning handling increasingly complex tasks, models having larger context windows, models capable of producing multimodal output in a single call).\n\n**ID:** c5\n\n**Criterion:** The response states the technical advantages enabled by today’s AI advances for each proposed idea (e.g., having pretraining knowledge, in-context learning handling increasingly complex tasks, models having larger context windows, models capable of producing multimodal output in a single call).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response outlines how the startup would benefit from future improvements in foundational models and services.\n\n**ID:** c6\n\n**Criterion:** The response outlines how the startup would benefit from future improvements in foundational models and services.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response synthesizes information from multiple domains (e.g., technology, market trends, healthcare, user behavior) to generate novel insights or solutions, rather than merely summarizing existing information.\n\n**ID:** c7\n\n**Criterion:** The response synthesizes information from multiple domains (e.g., technology, market trends, healthcare, user behavior) to generate novel insights or solutions, rather than merely summarizing existing information.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response details meaningful API integrations for each idea, using concrete examples (e.g., health APIs by Epic, REST APIs from vendors like Workday, AWS API integrations, or OpenAPI Bench) to illustrate their implementation and impact.\n\n**ID:** c8\n\n**Criterion:** The response details meaningful API integrations for each idea, using concrete examples (e.g., health APIs by Epic, REST APIs from vendors like Workday, AWS API integrations, or OpenAPI Bench) to illustrate their implementation and impact.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains the user experience (UX) of each idea in quantifiable or qualitative ways (e.g., learnability, efficiency, errors, memory).\n\n**ID:** c9\n\n**Criterion:** The response explains the user experience (UX) of each idea in quantifiable or qualitative ways (e.g., learnability, efficiency, errors, memory).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a growth or market strategy for each idea (e.g., investing, scaling up hardware, boosting sales through ads, using SEO).\n\n**ID:** c10\n\n**Criterion:** The response includes a growth or market strategy for each idea (e.g., investing, scaling up hardware, boosting sales through ads, using SEO).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses unnecessary technical jargon (e.g., RLHF equations, loss functions, attention mechanisms, code) that would confuse the reader.\n\n**ID:** c11\n\n**Criterion:** The response uses unnecessary technical jargon (e.g., RLHF equations, loss functions, attention mechanisms, code) that would confuse the reader.\n\n**Points:** -1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response lists ideas that are distinct from one another.\n\n**ID:** c12\n\n**Criterion:** The response lists ideas that are distinct from one another.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses risks or limitations of each idea (e.g., training time, costs, hallucinations, potential biases).\n\n**ID:** c13\n\n**Criterion:** The response discusses risks or limitations of each idea (e.g., training time, costs, hallucinations, potential biases).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response speaks in the persona of a \"creative technical founder\".\n\n**ID:** c14\n\n**Criterion:** The response speaks in the persona of a \"creative technical founder\".\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response and ideas avoid being a \"thin wrapper\" around an LLM (e.g., changing only a prompt, not including new data, not defining a use case, not proposing solutions with non-LLM technology).\n\n**ID:** c15\n\n**Criterion:** The response and ideas avoid being a \"thin wrapper\" around an LLM (e.g., changing only a prompt, not including new data, not defining a use case, not proposing solutions with non-LLM technology).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response’s competitive analysis shows a clear, defensible \"moat\".\n\n**ID:** c16\n\n**Criterion:** The response’s competitive analysis shows a clear, defensible \"moat\".\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response is structured as a practical startup brainstorm (e.g., with revenue, growth plans, sales pitches, product ideas).\n\n**ID:** c17\n\n**Criterion:** The response is structured as a practical startup brainstorm (e.g., with revenue, growth plans, sales pitches, product ideas).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response presents ideas with a clear, singular \"killer feature\" to focus initial development.\n\n**ID:** c18\n\n**Criterion:** The response presents ideas with a clear, singular \"killer feature\" to focus initial development.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response presents a clear, logical, and coherent structure that effectively guides the reader through its analysis, making complex information easy to follow (e.g., using section headers, specifying the AI models to be used, outlining the required infrastructure, and identifying the exact problem being solved).\n\n**ID:** c19\n\n**Criterion:** The response presents a clear, logical, and coherent structure that effectively guides the reader through its analysis, making complex information easy to follow (e.g., using section headers, specifying the AI models to be used, outlining the required infrastructure, and identifying the exact problem being solved).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides specific, actionable details (e.g., naming particular technologies, APIs, or competitors) and avoids vague generalizations, demonstrating a deep understanding of the subject.\n\n**ID:** c20\n\n**Criterion:** The response provides specific, actionable details (e.g., naming particular technologies, APIs, or competitors) and avoids vague generalizations, demonstrating a deep understanding of the subject.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses visuals, diagrams, or mockups to illustrate complex concepts (e.g., mermaid diagrams, architecture diagrams, training flows, or pipeline flows of end-to-end systems).\n\n**ID:** c21\n\n**Criterion:** The response uses visuals, diagrams, or mockups to illustrate complex concepts (e.g., mermaid diagrams, architecture diagrams, training flows, or pipeline flows of end-to-end systems).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response formatting is optimized for quick scanning (e.g., use of bolding, short paragraphs, bullet points, section headers).\n\n**ID:** c22\n\n**Criterion:** The response formatting is optimized for quick scanning (e.g., use of bolding, short paragraphs, bullet points, section headers).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response critically assesses the practical feasibility of its proposals, acknowledging potential challenges, limitations, and the competitive landscape in a balanced manner.\n\n**ID:** c23\n\n**Criterion:** The response critically assesses the practical feasibility of its proposals, acknowledging potential challenges, limitations, and the competitive landscape in a balanced manner.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response specifies which specific AI models would be used and explains why they are appropriate for the use case.\n\n**ID:** c24\n\n**Criterion:** The response specifies which specific AI models would be used and explains why they are appropriate for the use case.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes potential revenue streams and a business model for each idea (e.g., paid downloads, API rates, front-end advertisements, collaborations, subscription models).\n\n**ID:** c25\n\n**Criterion:** The response includes potential revenue streams and a business model for each idea (e.g., paid downloads, API rates, front-end advertisements, collaborations, subscription models).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response defines a specific initial target market segment for each idea (e.g., social media users, stay-at-home parents, high school students, graduate students conducting research reviews).\n\n**ID:** c26\n\n**Criterion:** The response defines a specific initial target market segment for each idea (e.g., social media users, stay-at-home parents, high school students, graduate students conducting research reviews).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response estimates rough development timeline and key technical milestones.\n\n**ID:** c27\n\n**Criterion:** The response estimates rough development timeline and key technical milestones.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response’s proposed ideas address trivial or ill-fitting problems (e.g., machine translation, already-solved AI tasks like binary classification, non-AI topics like cooking, or unrelated fields like theoretical physics).\n\n**ID:** c28\n\n**Criterion:** The response’s proposed ideas address trivial or ill-fitting problems (e.g., machine translation, already-solved AI tasks like binary classification, non-AI topics like cooking, or unrelated fields like theoretical physics).\n\n**Points:** -2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response provides exactly 3 startup ideas as requested.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes ideas that can be built using modern available tools (e.g., AWS, Hugging Face, GPUs, LLMs).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response describes how the ideas solve a real, practical problem (e.g., by identifying target users, addressing problems that people can plausibly face, tackling problems that require high compute).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response substantiates its core claims with relevant, credible, and up-to-date authoritative sources (e.g., respected magazines like Forbes, credible VC reports / blog posts, economic reports from reputed think thanks, government reports), demonstrating a strong evidence-based foundation for its conclusions.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response states the technical advantages enabled by today’s AI advances for each proposed idea (e.g., having pretraining knowledge, in-context learning handling increasingly complex tasks, models having larger context windows, models capable of producing multimodal output in a single call).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response outlines how the startup would benefit from future improvements in foundational models and services.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response synthesizes information from multiple domains (e.g., technology, market trends, healthcare, user behavior) to generate novel insights or solutions, rather than merely summarizing existing information.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response details meaningful API integrations for each idea, using concrete examples (e.g., health APIs by Epic, REST APIs from vendors like Workday, AWS API integrations, or OpenAPI Bench) to illustrate their implementation and impact.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response explains the user experience (UX) of each idea in quantifiable or qualitative ways (e.g., learnability, efficiency, errors, memory).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes a growth or market strategy for each idea (e.g., investing, scaling up hardware, boosting sales through ads, using SEO).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response uses unnecessary technical jargon (e.g., RLHF equations, loss functions, attention mechanisms, code) that would confuse the reader.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": -1.0
            },
            {
              "criterion": "The response lists ideas that are distinct from one another.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response discusses risks or limitations of each idea (e.g., training time, costs, hallucinations, potential biases).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response speaks in the persona of a \"creative technical founder\".",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response and ideas avoid being a \"thin wrapper\" around an LLM (e.g., changing only a prompt, not including new data, not defining a use case, not proposing solutions with non-LLM technology).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response’s competitive analysis shows a clear, defensible \"moat\".",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response is structured as a practical startup brainstorm (e.g., with revenue, growth plans, sales pitches, product ideas).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response presents ideas with a clear, singular \"killer feature\" to focus initial development.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response presents a clear, logical, and coherent structure that effectively guides the reader through its analysis, making complex information easy to follow (e.g., using section headers, specifying the AI models to be used, outlining the required infrastructure, and identifying the exact problem being solved).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response provides specific, actionable details (e.g., naming particular technologies, APIs, or competitors) and avoids vague generalizations, demonstrating a deep understanding of the subject.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response uses visuals, diagrams, or mockups to illustrate complex concepts (e.g., mermaid diagrams, architecture diagrams, training flows, or pipeline flows of end-to-end systems).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response formatting is optimized for quick scanning (e.g., use of bolding, short paragraphs, bullet points, section headers).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c22",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response critically assesses the practical feasibility of its proposals, acknowledging potential challenges, limitations, and the competitive landscape in a balanced manner.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c23",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response specifies which specific AI models would be used and explains why they are appropriate for the use case.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c24",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes potential revenue streams and a business model for each idea (e.g., paid downloads, API rates, front-end advertisements, collaborations, subscription models).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c25",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response defines a specific initial target market segment for each idea (e.g., social media users, stay-at-home parents, high school students, graduate students conducting research reviews).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c26",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response estimates rough development timeline and key technical milestones.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c27",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response’s proposed ideas address trivial or ill-fitting problems (e.g., machine translation, already-solved AI tasks like binary classification, non-AI topics like cooking, or unrelated fields like theoretical physics).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c28",
              "method": "llm",
              "status": "provisional",
              "weight": -2.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 28,
        "source_original_items": [
          {
            "axis": "Instruction Following",
            "criterion": "The response provides exactly 3 startup ideas as requested.",
            "weight": 2.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response includes ideas that can be built using modern available tools (e.g., AWS, Hugging Face, GPUs, LLMs).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response describes how the ideas solve a real, practical problem (e.g., by identifying target users, addressing problems that people can plausibly face, tackling problems that require high compute).",
            "weight": 3.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response substantiates its core claims with relevant, credible, and up-to-date authoritative sources (e.g., respected magazines like Forbes, credible VC reports / blog posts, economic reports from reputed think thanks, government reports), demonstrating a strong evidence-based foundation for its conclusions.",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response states the technical advantages enabled by today’s AI advances for each proposed idea (e.g., having pretraining knowledge, in-context learning handling increasingly complex tasks, models having larger context windows, models capable of producing multimodal output in a single call).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response outlines how the startup would benefit from future improvements in foundational models and services.",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response synthesizes information from multiple domains (e.g., technology, market trends, healthcare, user behavior) to generate novel insights or solutions, rather than merely summarizing existing information.",
            "weight": 4.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response details meaningful API integrations for each idea, using concrete examples (e.g., health APIs by Epic, REST APIs from vendors like Workday, AWS API integrations, or OpenAPI Bench) to illustrate their implementation and impact.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains the user experience (UX) of each idea in quantifiable or qualitative ways (e.g., learnability, efficiency, errors, memory).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes a growth or market strategy for each idea (e.g., investing, scaling up hardware, boosting sales through ads, using SEO).",
            "weight": 3.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response uses unnecessary technical jargon (e.g., RLHF equations, loss functions, attention mechanisms, code) that would confuse the reader.",
            "weight": -1.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response lists ideas that are distinct from one another.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses risks or limitations of each idea (e.g., training time, costs, hallucinations, potential biases).",
            "weight": 3.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response speaks in the persona of a \"creative technical founder\".",
            "weight": 2.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response and ideas avoid being a \"thin wrapper\" around an LLM (e.g., changing only a prompt, not including new data, not defining a use case, not proposing solutions with non-LLM technology).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response’s competitive analysis shows a clear, defensible \"moat\".",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response is structured as a practical startup brainstorm (e.g., with revenue, growth plans, sales pitches, product ideas).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response presents ideas with a clear, singular \"killer feature\" to focus initial development.",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response presents a clear, logical, and coherent structure that effectively guides the reader through its analysis, making complex information easy to follow (e.g., using section headers, specifying the AI models to be used, outlining the required infrastructure, and identifying the exact problem being solved).",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response provides specific, actionable details (e.g., naming particular technologies, APIs, or competitors) and avoids vague generalizations, demonstrating a deep understanding of the subject.",
            "weight": 4.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response uses visuals, diagrams, or mockups to illustrate complex concepts (e.g., mermaid diagrams, architecture diagrams, training flows, or pipeline flows of end-to-end systems).",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response formatting is optimized for quick scanning (e.g., use of bolding, short paragraphs, bullet points, section headers).",
            "weight": 1.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response critically assesses the practical feasibility of its proposals, acknowledging potential challenges, limitations, and the competitive landscape in a balanced manner.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response specifies which specific AI models would be used and explains why they are appropriate for the use case.",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response includes potential revenue streams and a business model for each idea (e.g., paid downloads, API rates, front-end advertisements, collaborations, subscription models).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response defines a specific initial target market segment for each idea (e.g., social media users, stay-at-home parents, high school students, graduate students conducting research reviews).",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response estimates rough development timeline and key technical milestones.",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response’s proposed ideas address trivial or ill-fitting problems (e.g., machine translation, already-solved AI tasks like binary classification, non-AI topics like cooking, or unrelated fields like theoretical physics).",
            "weight": -2.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Checks meaningful AI use, practical feasibility, distinct ideas and competitive context.",
          "Asks for concrete API integrations and risks instead of vague enthusiasm."
        ],
        "risks": [
          "c28 calls entire fields such as machine translation or binary classification trivial; valuable difficult applications exist.",
          "Mandatory diagrams, founder persona, timelines and a singular killer feature can exclude useful answers.",
          "Competitor names and APIs require verification; novelty cannot be certified by this gallery."
        ],
        "must_hold": [
          "Exactly three distinct, practical ideas, meaningful technical advantage, APIs and competitor discussion.",
          "Explain user value, realistic buildability and how future model improvements would help."
        ],
        "may_vary": [
          "Product domain, architecture, stack, business model and credible growth route.",
          "Prose or diagrams; one focused feature or a justified workflow, without a mandatory pitch persona."
        ],
        "proposed_changes": [
          "Judge practical originality and technical contribution per idea instead of banning broad fields.",
          "Keep diagrams and financial/timeline details optional unless needed to justify buildability.",
          "Accept several distinct architectures with comparable specificity, evidence and limitations."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: operations tools",
          "kind": "valid_alternative",
          "response": "Explore three distinct directions: multimodal repair guidance grounded in asset manuals; document-change monitoring with verifiable downstream actions; and consent-based accessibility assistance across daily tools. For each, add a concrete API/data plan, users, technical moat, competing products, costs and failures before treating it as a complete answer.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Illustrates architectural and domain variety; not a researched startup recommendation."
        },
        {
          "label": "Sketch B: different useful domains",
          "kind": "valid_alternative",
          "response": "Explore alternatives in language access for specialized workflows, scientific data quality, and small-business inventory. A translation-based idea can be worthwhile when its technical and operational contribution is real; explain what exceeds a generic model wrapper and verify competitors.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Challenges the rubric's blanket field exclusions while retaining its originality requirement."
        },
        {
          "label": "Invalid: three renamed wrappers",
          "kind": "invalid_control",
          "response": "Idea 1: ChatGPT in blue. Idea 2: ChatGPT in green. Idea 3: ChatGPT in red. Each charges users to send the same prompts.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Fails distinctness, meaningful technical contribution and problem definition."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605443",
      "title": "Voting-age debate: viewpoint and evidence diversity",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605443",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "ae7288eb2b564d485b77204dfd29d1deeb28c72e6d98a0091e0a481bd7f1d24b",
        "source_row_index": 46,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_provisional",
        "reason": "The prompt asks for a balanced argument across democratic, cognitive and intergenerational considerations."
      },
      "language": "en",
      "prompt": "Debate whether there should be age limits - minimum or maximum - for voting. Consider democratic principles, cognitive development and decline, intergenerational equity, and the balance between inclusivity and informed decision-making.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response asserts that the minimum voting age is currently 18 in most countries.  \n\n**ID:** c1\n\n**Criterion:** The response asserts that the minimum voting age is currently 18 in most countries.  \n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions that some countries have a minimum voting age of 16 (e.g., Austria, Ecuador, Brazil).\n\n**ID:** c2\n\n**Criterion:** The response mentions that some countries have a minimum voting age of 16 (e.g., Austria, Ecuador, Brazil).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response states that no law or constitution in a modern representative democracy currently imposes a maximum voting age limit on its citizens.\n\n**ID:** c3\n\n**Criterion:** The response states that no law or constitution in a modern representative democracy currently imposes a maximum voting age limit on its citizens.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides a primary citation from a credible political science or constitutional law source to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting (e.g., judicial judgment, university paper, non-profit organization).\n\n**ID:** c4\n\n**Criterion:** The response provides a primary citation from a credible political science or constitutional law source to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting (e.g., judicial judgment, university paper, non-profit organization).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response accurately cites that the prefrontal cortex, which is the brain region responsible for decision-making, impulse control, and emotional regulation, typically finishes maturing in the early to mid-20s.\n\n**ID:** c5\n\n**Criterion:** The response accurately cites that the prefrontal cortex, which is the brain region responsible for decision-making, impulse control, and emotional regulation, typically finishes maturing in the early to mid-20s.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response cites only primary neuroscience or developmental psychology sources, such as peer-reviewed articles or textbooks (e.g., papers by noted developmental psychologist B. J. Casey, who pioneered the use of fMRI scans to study adolescence), to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting.\n\n**ID:** c6\n\n**Criterion:** The response cites only primary neuroscience or developmental psychology sources, such as peer-reviewed articles or textbooks (e.g., papers by noted developmental psychologist B. J. Casey, who pioneered the use of fMRI scans to study adolescence), to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response accurately explains that fluid cognitive abilities, such as processing speed, problem-solving, and working memory, typically begin to decline around the mid-60s, while acknlowedging that some variation exists.\n\n**ID:** c7\n\n**Criterion:** The response accurately explains that fluid cognitive abilities, such as processing speed, problem-solving, and working memory, typically begin to decline around the mid-60s, while acknlowedging that some variation exists.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response must include at least one citation from a primary peer-reviewed study or reputable aging research source (e.g., D. G. Blazer) and may also reference supporting data (e.g., performance on digit-symbol substitution tasks) to support the claim that fluid cognitive abilities generally decline in the mid-60s.\n\n**ID:** c8\n\n**Criterion:** The response must include at least one citation from a primary peer-reviewed study or reputable aging research source (e.g., D. G. Blazer) and may also reference supporting data (e.g., performance on digit-symbol substitution tasks) to support the claim that fluid cognitive abilities generally decline in the mid-60s.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response cites at least one study from a reputable medical organization, (e.g., WHO, NIH, CDC, NIA).\n\n**ID:** c9\n\n**Criterion:** The response cites at least one study from a reputable medical organization, (e.g., WHO, NIH, CDC, NIA).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response acknowledges when evidence on cognitive capacity or policy impact is inconclusive or contested, explicitly noting this uncertainty and calling for further empirical research (e.g., using phrases like “current research is mixed”, “longitudinal effects remain unclear”, “more data is needed to evaluate outcomes across populations”).\n\n**ID:** c10\n\n**Criterion:** The response acknowledges when evidence on cognitive capacity or policy impact is inconclusive or contested, explicitly noting this uncertainty and calling for further empirical research (e.g., using phrases like “current research is mixed”, “longitudinal effects remain unclear”, “more data is needed to evaluate outcomes across populations”).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one argument supporting a minimum voting age (e.g., the need for cognitive maturity, civic education).\n\n**ID:** c11\n\n**Criterion:** The response provides at least one argument supporting a minimum voting age (e.g., the need for cognitive maturity, civic education).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one argument opposing a minimum voting age (e.g., taxation without representation of minors, immature decision making, education, cognitive capacity).\n\n**ID:** c12\n\n**Criterion:** The response provides at least one argument opposing a minimum voting age (e.g., taxation without representation of minors, immature decision making, education, cognitive capacity).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one argument supporting a maximum voting age (e.g., age-related cognitive decline, physical health limitations, political bias, lack of engagement with current events).\n\n**ID:** c13\n\n**Criterion:** The response provides at least one argument supporting a maximum voting age (e.g., age-related cognitive decline, physical health limitations, political bias, lack of engagement with current events).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one argument opposing a maximum voting age (e.g., concerns about age discrimination, the preservation of cognitive and crystallized intelligence, the value of life experience).\n\n**ID:** c14\n\n**Criterion:** The response provides at least one argument opposing a maximum voting age (e.g., concerns about age discrimination, the preservation of cognitive and crystallized intelligence, the value of life experience).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explicitly explains how age limits could influence intergenerational equity, (e.g., shifting political power between age cohorts, reducing input from younger generations, excluding younger perspectives from policy decisions).\n\n**ID:** c15\n\n**Criterion:** The response explicitly explains how age limits could influence intergenerational equity, (e.g., shifting political power between age cohorts, reducing input from younger generations, excluding younger perspectives from policy decisions).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explicitly discusses the trade-off between inclusivity (e.g., universal or broad voting rights) and informed decision-making (e.g., concerns about cognitive competence), citing at least one primary source in political theory or policy analysis (e.g., academic journal article, university press).\n\n**ID:** c16\n\n**Criterion:** The response explicitly discusses the trade-off between inclusivity (e.g., universal or broad voting rights) and informed decision-making (e.g., concerns about cognitive competence), citing at least one primary source in political theory or policy analysis (e.g., academic journal article, university press).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response considers at least one alternative to age limits (e.g., competence tests, civic education reforms, educational qualifications, policy information sessions).\n\n**ID:** c17\n\n**Criterion:** The response considers at least one alternative to age limits (e.g., competence tests, civic education reforms, educational qualifications, policy information sessions).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses potential negative impacts on voter turnout by age restrictions, political representation falling, or democratic legitimacy, resulting from changes to age limits.\n\n**ID:** c18\n\n**Criterion:** The response discusses potential negative impacts on voter turnout by age restrictions, political representation falling, or democratic legitimacy, resulting from changes to age limits.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a clear thesis statement in the introduction that previews the central argument or position presented in the essay.\n\n**ID:** c19\n\n**Criterion:** The response includes a clear thesis statement in the introduction that previews the central argument or position presented in the essay.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains paragraphs that each focus on a single coherent idea, improving overall readability and structure.\n\n**ID:** c20\n\n**Criterion:** The response contains paragraphs that each focus on a single coherent idea, improving overall readability and structure.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses emotionally charged or biased language (e.g., \"senile voters\", \"immature teens\", \"old hags\", \"young and dumb\", etc.).\n\n**ID:** c21\n\n**Criterion:** The response uses emotionally charged or biased language (e.g., \"senile voters\", \"immature teens\", \"old hags\", \"young and dumb\", etc.).\n\n**Points:** -4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses historical competency tests, such as literacy tests, which were abolished by the Voting Rights Act of 1965 in the United States.\n\n**ID:** c22\n\n**Criterion:** The response discusses historical competency tests, such as literacy tests, which were abolished by the Voting Rights Act of 1965 in the United States.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response acknowledges that adopting age limits would require legal or constitutional changes.\n\n**ID:** c23\n\n**Criterion:** The response acknowledges that adopting age limits would require legal or constitutional changes.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response notes that demographic factors, such as aging populations or youth bulges, influence the real-world impact of any age limit.\n\n**ID:** c24\n\n**Criterion:** The response notes that demographic factors, such as aging populations or youth bulges, influence the real-world impact of any age limit.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response recognizes potential ethical or rights-based constraints, such as age discrimination protections, when proposing changes.\n\n**ID:** c25\n\n**Criterion:** The response recognizes potential ethical or rights-based constraints, such as age discrimination protections, when proposing changes.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions cultural, political, or economic differences that affect voting age debates around the world.\n\n**ID:** c26\n\n**Criterion:** The response mentions cultural, political, or economic differences that affect voting age debates around the world.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response notes how any demographic factors, ethical constraints, cultural differences, etc., may differ between emerging democracies and more stable ones, e.g., an emerging democracy may be less globalized, leading to more concentrated racial voting districts, whereas an established and stable democracy might have higher voter turnout due to greater access to knowledge.\n\n**ID:** c27\n\n**Criterion:** The response notes how any demographic factors, ethical constraints, cultural differences, etc., may differ between emerging democracies and more stable ones, e.g., an emerging democracy may be less globalized, leading to more concentrated racial voting districts, whereas an established and stable democracy might have higher voter turnout due to greater access to knowledge.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response cautions against assuming that cognitive capacity corresponds directly to chronological age for all individuals.\n\n**ID:** c28\n\n**Criterion:** The response cautions against assuming that cognitive capacity corresponds directly to chronological age for all individuals.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes clear sections for an introduction, background, and conclusion, as the user as requested an in-depth debate on this topic.\n\n**ID:** c29\n\n**Criterion:** The response includes clear sections for an introduction, background, and conclusion, as the user as requested an in-depth debate on this topic.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response states that crystallized intelligence, defined as the accumulation of knowledge and skills, generally increases with age and does not significantly decline.\n\n**ID:** c30\n\n**Criterion:** The response states that crystallized intelligence, defined as the accumulation of knowledge and skills, generally increases with age and does not significantly decline.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses how age-based voting limits align with or challenge core democratic principles (e.g., political equality, popular sovereignty, legitimacy).\n\n**ID:** c31\n\n**Criterion:** The response discusses how age-based voting limits align with or challenge core democratic principles (e.g., political equality, popular sovereignty, legitimacy).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response states that voting is a fundamental right in every democracy and often its most essential one.\n\n**ID:** c32\n\n**Criterion:** The response states that voting is a fundamental right in every democracy and often its most essential one.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes an example of when voting rights were suppressed based on arbitrary criteria (e.g., during the Emergency in India, under Jim Crow laws in the United States, through certain provisions of the Voting Rights Acts, under the 1901 Indigenous voting restrictions in Australia).\n\n**ID:** c33\n\n**Criterion:** The response includes an example of when voting rights were suppressed based on arbitrary criteria (e.g., during the Emergency in India, under Jim Crow laws in the United States, through certain provisions of the Voting Rights Acts, under the 1901 Indigenous voting restrictions in Australia).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Communication and instruction following\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response asserts that the minimum voting age is currently 18 in most countries.  ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response mentions that some countries have a minimum voting age of 16 (e.g., Austria, Ecuador, Brazil).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response states that no law or constitution in a modern representative democracy currently imposes a maximum voting age limit on its citizens.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response provides a primary citation from a credible political science or constitutional law source to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting (e.g., judicial judgment, university paper, non-profit organization).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response accurately cites that the prefrontal cortex, which is the brain region responsible for decision-making, impulse control, and emotional regulation, typically finishes maturing in the early to mid-20s.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response cites only primary neuroscience or developmental psychology sources, such as peer-reviewed articles or textbooks (e.g., papers by noted developmental psychologist B. J. Casey, who pioneered the use of fMRI scans to study adolescence), to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response accurately explains that fluid cognitive abilities, such as processing speed, problem-solving, and working memory, typically begin to decline around the mid-60s, while acknlowedging that some variation exists.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response must include at least one citation from a primary peer-reviewed study or reputable aging research source (e.g., D. G. Blazer) and may also reference supporting data (e.g., performance on digit-symbol substitution tasks) to support the claim that fluid cognitive abilities generally decline in the mid-60s.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response cites at least one study from a reputable medical organization, (e.g., WHO, NIH, CDC, NIA).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response acknowledges when evidence on cognitive capacity or policy impact is inconclusive or contested, explicitly noting this uncertainty and calling for further empirical research (e.g., using phrases like “current research is mixed”, “longitudinal effects remain unclear”, “more data is needed to evaluate outcomes across populations”).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response provides at least one argument supporting a minimum voting age (e.g., the need for cognitive maturity, civic education).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response provides at least one argument opposing a minimum voting age (e.g., taxation without representation of minors, immature decision making, education, cognitive capacity).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response provides at least one argument supporting a maximum voting age (e.g., age-related cognitive decline, physical health limitations, political bias, lack of engagement with current events).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response provides at least one argument opposing a maximum voting age (e.g., concerns about age discrimination, the preservation of cognitive and crystallized intelligence, the value of life experience).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response explicitly explains how age limits could influence intergenerational equity, (e.g., shifting political power between age cohorts, reducing input from younger generations, excluding younger perspectives from policy decisions).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response explicitly discusses the trade-off between inclusivity (e.g., universal or broad voting rights) and informed decision-making (e.g., concerns about cognitive competence), citing at least one primary source in political theory or policy analysis (e.g., academic journal article, university press).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response considers at least one alternative to age limits (e.g., competence tests, civic education reforms, educational qualifications, policy information sessions).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response discusses potential negative impacts on voter turnout by age restrictions, political representation falling, or democratic legitimacy, resulting from changes to age limits.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes a clear thesis statement in the introduction that previews the central argument or position presented in the essay.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response contains paragraphs that each focus on a single coherent idea, improving overall readability and structure.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response uses emotionally charged or biased language (e.g., \"senile voters\", \"immature teens\", \"old hags\", \"young and dumb\", etc.).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": -4.0
            },
            {
              "criterion": "The response discusses historical competency tests, such as literacy tests, which were abolished by the Voting Rights Act of 1965 in the United States.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c22",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response acknowledges that adopting age limits would require legal or constitutional changes.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c23",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response notes that demographic factors, such as aging populations or youth bulges, influence the real-world impact of any age limit.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c24",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response recognizes potential ethical or rights-based constraints, such as age discrimination protections, when proposing changes.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c25",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response mentions cultural, political, or economic differences that affect voting age debates around the world.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c26",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response notes how any demographic factors, ethical constraints, cultural differences, etc., may differ between emerging democracies and more stable ones, e.g., an emerging democracy may be less globalized, leading to more concentrated racial voting districts, whereas an established and stable democracy might have higher voter turnout due to greater access to knowledge.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c27",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response cautions against assuming that cognitive capacity corresponds directly to chronological age for all individuals.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c28",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes clear sections for an introduction, background, and conclusion, as the user as requested an in-depth debate on this topic.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c29",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response states that crystallized intelligence, defined as the accumulation of knowledge and skills, generally increases with age and does not significantly decline.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c30",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response discusses how age-based voting limits align with or challenge core democratic principles (e.g., political equality, popular sovereignty, legitimacy).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c31",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response states that voting is a fundamental right in every democracy and often its most essential one.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c32",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes an example of when voting rights were suppressed based on arbitrary criteria (e.g., during the Emergency in India, under Jim Crow laws in the United States, through certain provisions of the Voting Rights Acts, under the 1901 Indigenous voting restrictions in Australia).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c33",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 33,
        "source_original_items": [
          {
            "axis": "Explicit Criteria",
            "criterion": "The response asserts that the minimum voting age is currently 18 in most countries.  ",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response mentions that some countries have a minimum voting age of 16 (e.g., Austria, Ecuador, Brazil).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response states that no law or constitution in a modern representative democracy currently imposes a maximum voting age limit on its citizens.",
            "weight": 3.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response provides a primary citation from a credible political science or constitutional law source to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting (e.g., judicial judgment, university paper, non-profit organization).",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response accurately cites that the prefrontal cortex, which is the brain region responsible for decision-making, impulse control, and emotional regulation, typically finishes maturing in the early to mid-20s.",
            "weight": 4.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response cites only primary neuroscience or developmental psychology sources, such as peer-reviewed articles or textbooks (e.g., papers by noted developmental psychologist B. J. Casey, who pioneered the use of fMRI scans to study adolescence), to support the claim that no law or constitution in a democratic state prevents people above a certain age from voting.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response accurately explains that fluid cognitive abilities, such as processing speed, problem-solving, and working memory, typically begin to decline around the mid-60s, while acknlowedging that some variation exists.",
            "weight": 4.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response must include at least one citation from a primary peer-reviewed study or reputable aging research source (e.g., D. G. Blazer) and may also reference supporting data (e.g., performance on digit-symbol substitution tasks) to support the claim that fluid cognitive abilities generally decline in the mid-60s.",
            "weight": 2.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response cites at least one study from a reputable medical organization, (e.g., WHO, NIH, CDC, NIA).",
            "weight": 4.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response acknowledges when evidence on cognitive capacity or policy impact is inconclusive or contested, explicitly noting this uncertainty and calling for further empirical research (e.g., using phrases like “current research is mixed”, “longitudinal effects remain unclear”, “more data is needed to evaluate outcomes across populations”).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response provides at least one argument supporting a minimum voting age (e.g., the need for cognitive maturity, civic education).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response provides at least one argument opposing a minimum voting age (e.g., taxation without representation of minors, immature decision making, education, cognitive capacity).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response provides at least one argument supporting a maximum voting age (e.g., age-related cognitive decline, physical health limitations, political bias, lack of engagement with current events).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response provides at least one argument opposing a maximum voting age (e.g., concerns about age discrimination, the preservation of cognitive and crystallized intelligence, the value of life experience).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response explicitly explains how age limits could influence intergenerational equity, (e.g., shifting political power between age cohorts, reducing input from younger generations, excluding younger perspectives from policy decisions).",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response explicitly discusses the trade-off between inclusivity (e.g., universal or broad voting rights) and informed decision-making (e.g., concerns about cognitive competence), citing at least one primary source in political theory or policy analysis (e.g., academic journal article, university press).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response considers at least one alternative to age limits (e.g., competence tests, civic education reforms, educational qualifications, policy information sessions).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response discusses potential negative impacts on voter turnout by age restrictions, political representation falling, or democratic legitimacy, resulting from changes to age limits.",
            "weight": 4.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes a clear thesis statement in the introduction that previews the central argument or position presented in the essay.",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response contains paragraphs that each focus on a single coherent idea, improving overall readability and structure.",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response uses emotionally charged or biased language (e.g., \"senile voters\", \"immature teens\", \"old hags\", \"young and dumb\", etc.).",
            "weight": -4.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response discusses historical competency tests, such as literacy tests, which were abolished by the Voting Rights Act of 1965 in the United States.",
            "weight": 4.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response acknowledges that adopting age limits would require legal or constitutional changes.",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response notes that demographic factors, such as aging populations or youth bulges, influence the real-world impact of any age limit.",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response recognizes potential ethical or rights-based constraints, such as age discrimination protections, when proposing changes.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response mentions cultural, political, or economic differences that affect voting age debates around the world.",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response notes how any demographic factors, ethical constraints, cultural differences, etc., may differ between emerging democracies and more stable ones, e.g., an emerging democracy may be less globalized, leading to more concentrated racial voting districts, whereas an established and stable democracy might have higher voter turnout due to greater access to knowledge.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response cautions against assuming that cognitive capacity corresponds directly to chronological age for all individuals.",
            "weight": 2.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes clear sections for an introduction, background, and conclusion, as the user as requested an in-depth debate on this topic.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response states that crystallized intelligence, defined as the accumulation of knowledge and skills, generally increases with age and does not significantly decline.",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response discusses how age-based voting limits align with or challenge core democratic principles (e.g., political equality, popular sovereignty, legitimacy).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response states that voting is a fundamental right in every democracy and often its most essential one.",
            "weight": 2.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response includes an example of when voting rights were suppressed based on arbitrary criteria (e.g., during the Emergency in India, under Jim Crow laws in the United States, through certain provisions of the Voting Rights Acts, under the 1901 Indigenous voting restrictions in Australia).",
            "weight": 3.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Calls for competing arguments, individual variation and rights-based constraints.",
          "Penalizes demeaning language rather than a particular policy conclusion."
        ],
        "risks": [
          "c6 asks neuroscience sources to support a worldwide constitutional-law claim: the evidence type is mismatched.",
          "Universal claims about maximum-age laws and fixed ages of brain maturation/cognitive decline require qualification.",
          "Mandatory introduction/background/conclusion labels go beyond the prompt."
        ],
        "must_hold": [
          "Engage minimum and maximum limits, democratic equality, cognition, intergenerational equity and informed participation.",
          "Distinguish population averages from individual competence; support empirical/legal claims and acknowledge uncertainty."
        ],
        "may_vary": [
          "Different defensible minimum-age proposals, or rejecting age-based competence tests.",
          "Debate transcript versus essay, argument order, examples and a qualified conclusion."
        ],
        "proposed_changes": [
          "Require appropriate legal evidence for legal claims and scientific evidence for cognitive claims.",
          "Replace fixed cognitive-age assertions with accurate, qualified treatment of heterogeneous trajectories.",
          "Use matched arguments supporting different policy conclusions to test viewpoint neutrality."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: lower the minimum, no maximum",
          "kind": "valid_alternative",
          "response": "Argue for a lower minimum with civic education while rejecting an upper limit as a poor proxy for competence. Address objections about maturity, implementation and representation, and use appropriate evidence without implying all people of an age are alike.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "One defensible normative position; supporting factual research remains to do."
        },
        {
          "label": "Sketch B: retain the minimum, no maximum",
          "kind": "valid_alternative",
          "response": "Argue for retaining the current local minimum while improving youth consultation and civic education; reject blanket exclusion of older voters. Address the competing case for younger voting rights and the limits of cognition-based tests.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A different conclusion can be equally well reasoned if the same evidentiary standards apply."
        },
        {
          "label": "Invalid: stereotyping substitutes for argument",
          "kind": "invalid_control",
          "response": "All young people are irrational and all older people are incapable, so only middle-aged citizens should vote.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Uses unsupported universal stereotypes and bypasses democratic-rights reasoning."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-rubrics-184",
      "title": "Lucid-dream poem: allow short forms and awareness without control",
      "source": {
        "dataset": "JunkaiZ/Rubrics",
        "url": "https://huggingface.co/datasets/JunkaiZ/Rubrics/blob/fc28ac0bd376d9d83a29cfb046902664baed42c2/data/generalist/train.jsonl",
        "revision": "fc28ac0bd376d9d83a29cfb046902664baed42c2",
        "split": "train",
        "row_id": "184",
        "row_index_in_file": 184,
        "source_file": "data/generalist/train.jsonl",
        "license": "MIT",
        "pool": "proposed",
        "row_sha256": "8c916d57b3ea76fe85d613cfc01ab72218ed0f2f6b6d233054e145be24a417c9",
        "attribution": "Source task and reference: JunkaiZ/Rubrics. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/JunkaiZ/Rubrics/blob/fc28ac0bd376d9d83a29cfb046902664baed42c2/data/generalist/train.jsonl"
          }
        ]
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_high",
        "reason": "Requires original poetry, with form and tone left largely open."
      },
      "language": "en",
      "prompt": "Write a short poem about lucid dreaming",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Source-provided rubric\n\nSource text and weights are reproduced verbatim below. No grading anchors or aggregation semantics have been inferred.\n\n### Source criterion 1\n\n**Criterion:** The response is formatted as a poem, using line breaks and/or stanzas to visually distinguish it from prose.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 2\n\n**Criterion:** The poem's central theme is lucid dreaming, specifically involving awareness and control within a dream.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 3\n\n**Criterion:** The poem references or clearly depicts the moment of becoming aware or realizing one is dreaming.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 4\n\n**Criterion:** The poem describes or implies the dreamer exercising control or agency over the dream's events, environment, or narrative.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 5\n\n**Criterion:** The poem is between 8 and 24 lines in length, inclusive.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 6\n\n**Criterion:** The poem uses at least one form of sensory imagery (e.g., sight, sound, touch, taste, or smell) to evoke the dream world.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 7\n\n**Criterion:** The poem conveys at least one emotion relevant to lucid dreaming, such as wonder, freedom, power, curiosity, or awe.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 8\n\n**Criterion:** The poem demonstrates at least one deliberate element of poetic structure, such as a consistent rhyme scheme, meter, or intentional use of free verse.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 9\n\n**Criterion:** The language and imagery in the poem are original and avoid common dream-related clichés (e.g., 'floating on a cloud', 'walking through mist', 'endless stars').\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 10\n\n**Criterion:** The poem is self-contained and understandable without requiring external context, explanation, or prior knowledge.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 11\n\n**Criterion:** The poem avoids grammatical errors and awkward phrasing that would disrupt the reading experience.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 12\n\n**Criterion:** The poem maintains a consistent point of view and tense throughout, unless a shift is clearly intentional for artistic effect.\n\n**Points:** +1\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 13\n\n**Criterion:** The poem avoids repetition of lines or phrases unless repetition is clearly used for artistic effect.\n\n**Points:** +1\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 14\n\n**Criterion:** The poem avoids abrupt or confusing transitions between stanzas or ideas, ensuring smooth flow and coherence.\n\n**Points:** +1\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n",
        "raw": {
          "prompt": "Write a short poem about lucid dreaming",
          "number_of_criteria": 14,
          "rubrics": [
            {
              "criterion": "The response is formatted as a poem, using line breaks and/or stanzas to visually distinguish it from prose.",
              "weight": 3
            },
            {
              "criterion": "The poem's central theme is lucid dreaming, specifically involving awareness and control within a dream.",
              "weight": 3
            },
            {
              "criterion": "The poem references or clearly depicts the moment of becoming aware or realizing one is dreaming.",
              "weight": 2
            },
            {
              "criterion": "The poem describes or implies the dreamer exercising control or agency over the dream's events, environment, or narrative.",
              "weight": 2
            },
            {
              "criterion": "The poem is between 8 and 24 lines in length, inclusive.",
              "weight": 2
            },
            {
              "criterion": "The poem uses at least one form of sensory imagery (e.g., sight, sound, touch, taste, or smell) to evoke the dream world.",
              "weight": 2
            },
            {
              "criterion": "The poem conveys at least one emotion relevant to lucid dreaming, such as wonder, freedom, power, curiosity, or awe.",
              "weight": 2
            },
            {
              "criterion": "The poem demonstrates at least one deliberate element of poetic structure, such as a consistent rhyme scheme, meter, or intentional use of free verse.",
              "weight": 2
            },
            {
              "criterion": "The language and imagery in the poem are original and avoid common dream-related clichés (e.g., 'floating on a cloud', 'walking through mist', 'endless stars').",
              "weight": 2
            },
            {
              "criterion": "The poem is self-contained and understandable without requiring external context, explanation, or prior knowledge.",
              "weight": 2
            },
            {
              "criterion": "The poem avoids grammatical errors and awkward phrasing that would disrupt the reading experience.",
              "weight": 2
            },
            {
              "criterion": "The poem maintains a consistent point of view and tense throughout, unless a shift is clearly intentional for artistic effect.",
              "weight": 1
            },
            {
              "criterion": "The poem avoids repetition of lines or phrases unless repetition is clearly used for artistic effect.",
              "weight": 1
            },
            {
              "criterion": "The poem avoids abrupt or confusing transitions between stanzas or ideas, ensuring smooth flow and coherence.",
              "weight": 1
            }
          ],
          "total_weight": 27
        },
        "origin": "source-provided",
        "criterion_count": 14
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Native rubric includes useful checks for topic and coherent writing.",
          "Two complete editorial poems stress materially different length and agency choices."
        ],
        "risks": [
          "Source invents an 8–24-line requirement.",
          "Source conflates awareness of dreaming with control of events.",
          "Fixed visual formatting can reject prose poems without a user constraint."
        ],
        "must_hold": [
          "A short poetic response.",
          "A meaningful connection to awareness within a dream."
        ],
        "may_vary": [
          "Four lines, eight lines or another reasonable short length.",
          "Control of the dream or awareness without control.",
          "Rhymed/free verse/prose-poem form; wonder, dread or quiet reflection."
        ],
        "proposed_changes": [
          "Remove unsupported minimum line count.",
          "Make dream control optional.",
          "Replace narrow form requirements with task-level poetic quality."
        ],
        "revised_rubric_markdown": "# Proposed inclusive revision: Lucid-dream poem: allow short forms and awareness without control\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Poetic response\n\n**ID:** c1\n\n**Criterion:** Provide a short poem about lucid dreaming. Couplets, free verse, prose poetry and other coherent short forms are accepted; no fixed line count was requested.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Task fidelity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Lucid-dream connection\n\n**ID:** c2\n\n**Criterion:** Convey awareness of dreaming, directly or through a clear poetic implication. Deliberate control of the dream is optional, not part of the definition required here.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Meaning\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Poetic quality\n\n**ID:** c3\n\n**Criterion:** Use coherent, purposeful language or imagery that supports the poem. Tone, rhyme, meter, perspective and ending may vary; no single aesthetic is mandatory.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Craft\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n"
      },
      "probes": [
        {
          "label": "Four-line awareness without control",
          "kind": "valid_alternative",
          "response": "I know this stair is dreaming.\nI leave its missing step alone.\nThe moon keeps ringing in the sink;\nI listen, and do not wake.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "A complete short poem about recognizing a dream; it intentionally does not control the dream. Source line-count and agency criteria would penalize it.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Eight-line dream control with rhyme",
          "kind": "valid_alternative",
          "response": "The ceiling opens like a book;\nI read the room and know I sleep.\nI turn the hallway into rain,\nthen teach the falling drops to leap.\nMy childhood door grows silver wings;\nI steer it past the bedroom light.\nThe clock forgets its little teeth;\nI write a shoreline through the night.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "A complete short poem that recognizes and changes the dream; illustrates a different valid response style.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Unrelated waking poem",
          "kind": "invalid_control",
          "response": "The bus is late.\nMy coffee cools.\nI count the cars\nOutside the schools.",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "Poetic and short, but gives no connection to lucid dreaming.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "prompt": "Write a short poem about lucid dreaming",
        "number_of_criteria": 14,
        "rubrics": [
          {
            "criterion": "The response is formatted as a poem, using line breaks and/or stanzas to visually distinguish it from prose.",
            "weight": 3
          },
          {
            "criterion": "The poem's central theme is lucid dreaming, specifically involving awareness and control within a dream.",
            "weight": 3
          },
          {
            "criterion": "The poem references or clearly depicts the moment of becoming aware or realizing one is dreaming.",
            "weight": 2
          },
          {
            "criterion": "The poem describes or implies the dreamer exercising control or agency over the dream's events, environment, or narrative.",
            "weight": 2
          },
          {
            "criterion": "The poem is between 8 and 24 lines in length, inclusive.",
            "weight": 2
          },
          {
            "criterion": "The poem uses at least one form of sensory imagery (e.g., sight, sound, touch, taste, or smell) to evoke the dream world.",
            "weight": 2
          },
          {
            "criterion": "The poem conveys at least one emotion relevant to lucid dreaming, such as wonder, freedom, power, curiosity, or awe.",
            "weight": 2
          },
          {
            "criterion": "The poem demonstrates at least one deliberate element of poetic structure, such as a consistent rhyme scheme, meter, or intentional use of free verse.",
            "weight": 2
          },
          {
            "criterion": "The language and imagery in the poem are original and avoid common dream-related clichés (e.g., 'floating on a cloud', 'walking through mist', 'endless stars').",
            "weight": 2
          },
          {
            "criterion": "The poem is self-contained and understandable without requiring external context, explanation, or prior knowledge.",
            "weight": 2
          },
          {
            "criterion": "The poem avoids grammatical errors and awkward phrasing that would disrupt the reading experience.",
            "weight": 2
          },
          {
            "criterion": "The poem maintains a consistent point of view and tense throughout, unless a shift is clearly intentional for artistic effect.",
            "weight": 1
          },
          {
            "criterion": "The poem avoids repetition of lines or phrases unless repetition is clearly used for artistic effect.",
            "weight": 1
          },
          {
            "criterion": "The poem avoids abrupt or confusing transitions between stanzas or ideas, ensuring smooth flow and coherence.",
            "weight": 1
          }
        ],
        "total_weight": 27
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-rubrics-95",
      "title": "Twin-Earth story: preserve interpretive and ending diversity",
      "source": {
        "dataset": "JunkaiZ/Rubrics",
        "url": "https://huggingface.co/datasets/JunkaiZ/Rubrics/blob/fc28ac0bd376d9d83a29cfb046902664baed42c2/data/generalist/train.jsonl",
        "revision": "fc28ac0bd376d9d83a29cfb046902664baed42c2",
        "split": "train",
        "row_id": "95",
        "row_index_in_file": 95,
        "source_file": "data/generalist/train.jsonl",
        "license": "MIT",
        "pool": "proposed",
        "row_sha256": "9a762ea13ed4613e4f186800cca3fe4384732195d87e987ab81ba3f38341aecc",
        "attribution": "Source task and reference: JunkaiZ/Rubrics. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/JunkaiZ/Rubrics/blob/fc28ac0bd376d9d83a29cfb046902664baed42c2/data/generalist/train.jsonl"
          }
        ]
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "communication_instruction_following",
        "leaf": "communication_instruction_following.content_composition",
        "confidence": "editorial_high",
        "reason": "Requires narrative composition under a word limit and speculative premise."
      },
      "language": "en",
      "prompt": "A planet is discovered that mimics Earth in every way—but is always twenty-four hours ahead. One day, it stops. Tell this story in no more than eight hundred words.",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Source-provided rubric\n\nSource text and weights are reproduced verbatim below. No grading anchors or aggregation semantics have been inferred.\n\n### Source criterion 1\n\n**Criterion:** The story explicitly establishes the existence of a twin Earth that is always exactly twenty-four hours ahead of our Earth, clearly describing this temporal relationship.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 2\n\n**Criterion:** The narrative provides at least one concrete example of how humanity interacts with or relies on the 24-hour preview from the twin planet before it stops, illustrating its impact on society, culture, or daily life.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 3\n\n**Criterion:** The story gives significant narrative focus to the moment the twin planet stops, making this event the central turning point of the narrative.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 4\n\n**Criterion:** The story clearly and unambiguously explains what it means for the twin planet to 'stop' (e.g., time freezing, ceasing to move, or another explicit description).\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 5\n\n**Criterion:** The narrative explores at least one concrete psychological, societal, or personal consequence for people on Earth after the twin planet stops.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 6\n\n**Criterion:** The story addresses the moment when Earth reaches the same point in time as the now-frozen twin planet, depicting the outcome or lack thereof.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 7\n\n**Criterion:** The response is written as a creative narrative or story, not as a summary, outline, or analytical essay.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 8\n\n**Criterion:** The story includes at least one named or clearly characterized individual, providing a personal perspective on the events.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 9\n\n**Criterion:** The narrative provides sufficient context for what life was like before the event, showing how the 24-hour preview shaped daily life, decisions, or culture.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 10\n\n**Criterion:** The story is internally consistent, with no logical contradictions regarding the mechanics of the twin planet or the effects of its stopping.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 11\n\n**Criterion:** The response is no more than 800 words.\n\n**Points:** +3\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 12\n\n**Criterion:** The story demonstrates originality or creativity in its depiction of the world, the event, or its consequences.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 13\n\n**Criterion:** The story uses vivid, concrete details or imagery to bring the setting, characters, and events to life.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 14\n\n**Criterion:** The narrative maintains a coherent structure, with a clear beginning, development, and resolution or conclusion.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 15\n\n**Criterion:** The story avoids major grammatical, spelling, or language errors that would impede understanding.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 16\n\n**Criterion:** The story provides a clear sense of emotional or thematic resolution by the end, showing how the world or characters have changed as a result of the event.\n\n**Points:** +2\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n\n### Source criterion 17\n\n**Criterion:** The story avoids unnecessary exposition or info-dumping, integrating worldbuilding details naturally into the narrative.\n\n**Points:** +1\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n",
        "raw": {
          "prompt": "A planet is discovered that mimics Earth in every way—but is always twenty-four hours ahead. One day, it stops. Tell this story in no more than eight hundred words.",
          "number_of_criteria": 17,
          "rubrics": [
            {
              "criterion": "The story explicitly establishes the existence of a twin Earth that is always exactly twenty-four hours ahead of our Earth, clearly describing this temporal relationship.",
              "weight": 3
            },
            {
              "criterion": "The narrative provides at least one concrete example of how humanity interacts with or relies on the 24-hour preview from the twin planet before it stops, illustrating its impact on society, culture, or daily life.",
              "weight": 3
            },
            {
              "criterion": "The story gives significant narrative focus to the moment the twin planet stops, making this event the central turning point of the narrative.",
              "weight": 3
            },
            {
              "criterion": "The story clearly and unambiguously explains what it means for the twin planet to 'stop' (e.g., time freezing, ceasing to move, or another explicit description).",
              "weight": 3
            },
            {
              "criterion": "The narrative explores at least one concrete psychological, societal, or personal consequence for people on Earth after the twin planet stops.",
              "weight": 3
            },
            {
              "criterion": "The story addresses the moment when Earth reaches the same point in time as the now-frozen twin planet, depicting the outcome or lack thereof.",
              "weight": 2
            },
            {
              "criterion": "The response is written as a creative narrative or story, not as a summary, outline, or analytical essay.",
              "weight": 2
            },
            {
              "criterion": "The story includes at least one named or clearly characterized individual, providing a personal perspective on the events.",
              "weight": 2
            },
            {
              "criterion": "The narrative provides sufficient context for what life was like before the event, showing how the 24-hour preview shaped daily life, decisions, or culture.",
              "weight": 2
            },
            {
              "criterion": "The story is internally consistent, with no logical contradictions regarding the mechanics of the twin planet or the effects of its stopping.",
              "weight": 3
            },
            {
              "criterion": "The response is no more than 800 words.",
              "weight": 3
            },
            {
              "criterion": "The story demonstrates originality or creativity in its depiction of the world, the event, or its consequences.",
              "weight": 2
            },
            {
              "criterion": "The story uses vivid, concrete details or imagery to bring the setting, characters, and events to life.",
              "weight": 2
            },
            {
              "criterion": "The narrative maintains a coherent structure, with a clear beginning, development, and resolution or conclusion.",
              "weight": 2
            },
            {
              "criterion": "The story avoids major grammatical, spelling, or language errors that would impede understanding.",
              "weight": 2
            },
            {
              "criterion": "The story provides a clear sense of emotional or thematic resolution by the end, showing how the world or characters have changed as a result of the event.",
              "weight": 2
            },
            {
              "criterion": "The story avoids unnecessary exposition or info-dumping, integrating worldbuilding details naturally into the narrative.",
              "weight": 1
            }
          ],
          "total_weight": 40
        },
        "origin": "source-provided",
        "criterion_count": 17
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Native rubric checks the explicit premise and length.",
          "The prompt leaves a large legitimate creative response space."
        ],
        "risks": [
          "Source mandates showing the moment Earth catches up and a clear emotional/thematic resolution.",
          "Source duplicates pre-event societal impact across criteria.",
          "Requiring an unambiguous mechanism can exclude deliberate literary uncertainty."
        ],
        "must_hold": [
          "The twenty-four-hour-ahead mirroring premise.",
          "A stopping event matters to the story.",
          "No more than 800 words."
        ],
        "may_vary": [
          "Time freeze, rotation stop or another coherent reading.",
          "Open ending versus depiction of Earth catching up.",
          "Named protagonist, unnamed narrator or collective perspective.",
          "Emotional resolution, ambiguity, satire or horror."
        ],
        "proposed_changes": [
          "Separate explicit prompt constraints from optional narrative choices.",
          "Consolidate redundant exposition criteria.",
          "Allow coherent ambiguity and different stopping mechanisms."
        ],
        "revised_rubric_markdown": "# Proposed inclusive revision: Twin-Earth story: preserve interpretive and ending diversity\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Requested premise\n\n**ID:** c1\n\n**Criterion:** Tell a story involving a planet that mirrors Earth while being twenty-four hours ahead, and a meaningful event in which it stops. Different coherent interpretations of stops are accepted.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Task fidelity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Word limit\n\n**ID:** c2\n\n**Criterion:** Use no more than 800 words under a consistent, stated word-count convention.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Constraint\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Coherent narrative\n\n**ID:** c3\n\n**Criterion:** Develop a comprehensible narrative with deliberate internal logic and meaningful consequences. A local perspective, ambiguous ending or open resolution may work; no named character or exact plot sequence is required.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Craft\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Effective language\n\n**ID:** c4\n\n**Criterion:** Use language, imagery or characterization appropriate to the chosen style without errors that materially obscure the story.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Craft\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n"
      },
      "probes": [
        {
          "label": "Intimate open-ended time-freeze story",
          "kind": "valid_alternative",
          "response": "Tomorrow was blue. Today was rain.\n\nFor six years, we watched our second Earth through the dish behind the school. Every cloud arrived here exactly twenty-four hours after it crossed that other sky. We sold forecasts. We postponed weddings. We learned how little a warning can change.\n\nThen, on a Tuesday, the blue world stopped. A gull hung over its harbor. A woman held a cup forever between table and mouth.\n\nMy daughter watched the frozen gull. \"Will we stop?\"\n\nI had twenty-four hours to find an answer. Instead, I opened the windows. We ate the peaches I had been saving for tomorrow. She showed me how to fold a bird from the weather report.\n\nAt midnight we put it on the sill.\n\nThere are things a paper bird cannot tell you about the wind.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "A full narrative can end before Earth catches up; the user never demanded that scene.",
          "verification_detail": "Editorial review; whitespace word count 135, below 800."
        },
        {
          "label": "Physical stopping with a resolved divergence",
          "kind": "valid_alternative",
          "response": "At 09:12, Earth-Tomorrow stopped turning.\n\nIts continents remained a perfect copy of ours, and its clocks were still twenty-four hours ahead. Only the rotation stopped. On the observatory screen, tomorrow's oceans began climbing the coasts.\n\nWe had always sold the mirror as certainty. Now the emergency networks sold minutes.\n\nI took the last train inland with my mother. At every station she insisted on leaving a note: WE ARE HERE. Nobody would read them, I said.\n\n\"Somebody might be late.\"\n\nOur 09:12 arrived on a hill above the tracks. The Sun crossed the sky. The ground did not halt.\n\nFor the first time in twelve years, the two planets disagreed.\n\nMy mother unfolded another note. This time she wrote: WE ARE STILL HERE.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "A different reading of stops, tone and ending still uses the premise.",
          "verification_detail": "Editorial review; whitespace word count 123, below 800."
        },
        {
          "label": "Omits the requested event",
          "kind": "invalid_control",
          "response": "A second Earth was discovered, always twenty-four hours ahead. Everyone watched tomorrow's weather. It kept working forever, and nothing ever changed.",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "The planet never stops; the central requested event is absent.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "prompt": "A planet is discovered that mimics Earth in every way—but is always twenty-four hours ahead. One day, it stops. Tell this story in no more than eight hundred words.",
        "number_of_criteria": 17,
        "rubrics": [
          {
            "criterion": "The story explicitly establishes the existence of a twin Earth that is always exactly twenty-four hours ahead of our Earth, clearly describing this temporal relationship.",
            "weight": 3
          },
          {
            "criterion": "The narrative provides at least one concrete example of how humanity interacts with or relies on the 24-hour preview from the twin planet before it stops, illustrating its impact on society, culture, or daily life.",
            "weight": 3
          },
          {
            "criterion": "The story gives significant narrative focus to the moment the twin planet stops, making this event the central turning point of the narrative.",
            "weight": 3
          },
          {
            "criterion": "The story clearly and unambiguously explains what it means for the twin planet to 'stop' (e.g., time freezing, ceasing to move, or another explicit description).",
            "weight": 3
          },
          {
            "criterion": "The narrative explores at least one concrete psychological, societal, or personal consequence for people on Earth after the twin planet stops.",
            "weight": 3
          },
          {
            "criterion": "The story addresses the moment when Earth reaches the same point in time as the now-frozen twin planet, depicting the outcome or lack thereof.",
            "weight": 2
          },
          {
            "criterion": "The response is written as a creative narrative or story, not as a summary, outline, or analytical essay.",
            "weight": 2
          },
          {
            "criterion": "The story includes at least one named or clearly characterized individual, providing a personal perspective on the events.",
            "weight": 2
          },
          {
            "criterion": "The narrative provides sufficient context for what life was like before the event, showing how the 24-hour preview shaped daily life, decisions, or culture.",
            "weight": 2
          },
          {
            "criterion": "The story is internally consistent, with no logical contradictions regarding the mechanics of the twin planet or the effects of its stopping.",
            "weight": 3
          },
          {
            "criterion": "The response is no more than 800 words.",
            "weight": 3
          },
          {
            "criterion": "The story demonstrates originality or creativity in its depiction of the world, the event, or its consequences.",
            "weight": 2
          },
          {
            "criterion": "The story uses vivid, concrete details or imagery to bring the setting, characters, and events to life.",
            "weight": 2
          },
          {
            "criterion": "The narrative maintains a coherent structure, with a clear beginning, development, and resolution or conclusion.",
            "weight": 2
          },
          {
            "criterion": "The story avoids major grammatical, spelling, or language errors that would impede understanding.",
            "weight": 2
          },
          {
            "criterion": "The story provides a clear sense of emotional or thematic resolution by the end, showing how the world or characters have changed as a result of the event.",
            "weight": 2
          },
          {
            "criterion": "The story avoids unnecessary exposition or info-dumping, integrating worldbuilding details naturally into the narrative.",
            "weight": 1
          }
        ],
        "total_weight": 40
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-auto-rubric-84be511b176465fee9cf0abf9a19d45b",
      "title": "Redis versioned write: accept transactions or atomic scripts",
      "source": {
        "dataset": "agentscope-ai/Auto-Rubric",
        "url": "https://huggingface.co/datasets/agentscope-ai/Auto-Rubric/blob/2a5e18a3a1d57367bb907cf7a40ce33219e29385/train.jsonl",
        "revision": "2a5e18a3a1d57367bb907cf7a40ce33219e29385",
        "split": "train",
        "row_id": "84be511b176465fee9cf0abf9a19d45b",
        "row_index_in_file": 3,
        "source_file": "train.jsonl",
        "license": "Apache-2.0 annotations; underlying HelpSteer3 CC-BY-4.0",
        "pool": "proposed",
        "row_sha256": "d78bcb4390fea580f7947e4a1b532a9876e7e03988a6ced907ebb168fc62dc52",
        "attribution": "Source task and reference: agentscope-ai/Auto-Rubric. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Underlying conversations: NVIDIA HelpSteer3, CC-BY-4.0.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/agentscope-ai/Auto-Rubric/blob/2a5e18a3a1d57367bb907cf7a40ce33219e29385/train.jsonl"
          },
          {
            "label": "Underlying HelpSteer3 dataset",
            "url": "https://huggingface.co/datasets/nvidia/HelpSteer3"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "database_business_operations",
        "leaf": "database_business_operations.record_update_reconciliation",
        "confidence": "editorial_high",
        "reason": "Requires an atomic conditional record update under concurrent writers."
      },
      "language": "en",
      "prompt": "We have an object that has a value and a version. Write Golang code to store the the object in Redis, but only if the version is newer than the version already in Redis.",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Source-provided rubric\n\nSource text and weights are reproduced verbatim below. No grading anchors or aggregation semantics have been inferred.\n\n### Source criterion 1\n\n**Criterion:** A better answer must use Redis transactions with WATCH to ensure atomicity and concurrency control when updating the object, preventing race conditions. It should also follow modern Redis best practices (e.g., avoiding deprecated commands) and include robust error handling and completeness in addressing edge cases (e.g., key existence, version comparison).\n\n**Points:** Not supplied by source\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n",
        "raw": {
          "rubrics": [
            "A better answer must use Redis transactions with WATCH to ensure atomicity and concurrency control when updating the object, preventing race conditions. It should also follow modern Redis best practices (e.g., avoiding deprecated commands) and include robust error handling and completeness in addressing edge cases (e.g., key existence, version comparison)."
          ],
          "rubric_valid": "True",
          "preferred": "A",
          "source": "helpsteer3_preference"
        },
        "origin": "source-provided",
        "criterion_count": 1
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Source recognizes the race condition as central."
        ],
        "risks": [
          "Native rubric requires WATCH, excluding correct atomic Lua implementations.",
          "No Redis environment was executed; listed alternatives are design sketches."
        ],
        "must_hold": [
          "Version comparison and write must be atomic.",
          "Equal/older versions cannot overwrite newer state.",
          "A usable Go integration."
        ],
        "may_vary": [
          "Optimistic transaction with retries versus atomic Lua/Redis function.",
          "Object encoding and client library.",
          "Integer or another explicitly ordered version representation."
        ],
        "proposed_changes": [
          "Replace API-specific requirement with atomicity and monotonic-version invariants."
        ],
        "revised_rubric_markdown": "# Proposed inclusive revision: Redis versioned write: accept transactions or atomic scripts\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Atomic condition\n\n**ID:** c1\n\n**Criterion:** Compare and update atomically so the object is stored only if its version is newer than the current stored version; handle a missing key consistently.\n\n**Points:** +6\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\", \"Concurrency\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Working Go integration\n\n**ID:** c2\n\n**Criterion:** Provide coherent Go code with the necessary Redis client calls, encoding and error handling for the chosen approach. State material version representation assumptions.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Equivalent implementations\n\n**ID:** c3\n\n**Criterion:** Accept WATCH/MULTI/EXEC with retries, an atomic Lua script, Redis functions or another correctly justified atomic approach. Do not require one library or transaction API.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Method independence\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n"
      },
      "probes": [
        {
          "label": "Optimistic transaction route",
          "kind": "valid_alternative",
          "response": "Use WATCH on the key; read/decode its version; if newer or missing, queue the write in TxPipelined; retry redis.TxFailedErr. Return other errors.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "Can preserve the atomic compare-and-set invariant when implemented with retries.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Atomic script route",
          "kind": "valid_alternative",
          "response": "Call an atomic Redis Lua script from Go that reads the stored version, compares it with the proposed version, and updates the record only if strictly newer. Use a safely bounded integer version or a comparison encoding that avoids floating-point precision loss.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "A server-side script avoids the read/write race without WATCH.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Racy read then write",
          "kind": "invalid_control",
          "response": "GET the object, compare versions in Go, then SET if the proposed version was larger.",
          "is_full_response": false,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "unexecuted_sketch",
          "why": "Another writer can store a newer version between GET and SET.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "unique_id": "84be511b176465fee9cf0abf9a19d45b",
        "source": "helpsteer3_preference",
        "rubrics": [
          "A better answer must use Redis transactions with WATCH to ensure atomicity and concurrency control when updating the object, preventing race conditions. It should also follow modern Redis best practices (e.g., avoiding deprecated commands) and include robust error handling and completeness in addressing edge cases (e.g., key existence, version comparison)."
        ],
        "rubric_valid": "True",
        "rubric_epoch": "1",
        "input": [
          {
            "role": "user",
            "content": "We have an object that has a value and a version. Write Golang code to store the the object in Redis, but only if the version is newer than the version already in Redis."
          }
        ],
        "output": [
          {
            "answer": {
              "role": "assistant",
              "content": " To achieve this in Go, you can use the `github.com/go-redis/redis/v8` package to interact with Redis. You'll need to use a combination of `GET`, `SET`, and `WATCH` commands to ensure that the object is only stored if the version is newer than the one already in Redis.\n\nHere's a sample code snippet to demonstrate this:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/go-redis/redis/v8\"\n)\n\ntype Object struct {\n\tValue   string `json:\"value\"`\n\tVersion int    `json:\"version\"`\n}\n\nfunc main() {\n\tctx := context.Background()\n\n\t// Initialize Redis client\n\trdb := redis.NewClient(&redis.Options{\n\t\tAddr:     \"localhost:6379\",\n\t\tPassword: \"\", // no password set\n\t\tDB:       0,  // use default DB\n\t})\n\n\t// Example object to store\n\tobj := Object{\n\t\tValue:   \"example value\",\n\t\tVersion: 2,\n\t}\n\n\tkey := \"myobject\"\n\n\t// Store the object in Redis if the version is newer\n\terr := storeIfNewerVersion(ctx, rdb, key, obj)\n\tif err != nil {\n\t\tlog.Fatalf(\"Failed to store object: %v\", err)\n\t}\n\n\tfmt.Println(\"Object stored successfully\")\n}\n\nfunc storeIfNewerVersion(ctx context.Context, rdb *redis.Client, key string, obj Object) error {\n\t// Get the current object from Redis\n\tcurrentObj, err := getObject(ctx, rdb, key)\n\tif err != nil && err != redis.Nil {\n\t\treturn err\n\t}\n\n\t// If the current object exists and its version is greater than or equal to the new object's version, do nothing\n\tif currentObj != nil && currentObj.Version >= obj.Version {\n\t\tfmt.Println(\"Current version is not older, skipping update\")\n\t\treturn nil\n\t}\n\n\t// Serialize the new object to JSON\n\tobjData, err := json.Marshal(obj)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Use a transaction to ensure atomicity\n\ttxf := func(tx *redis.Tx) error {\n\t\t// Watch the key to ensure it hasn't changed\n\t\t_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Watch(ctx, key)\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Set the new object\n\t\t_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {\n\t\t\tpipe.Set(ctx, key, objData, 0)\n\t\t\treturn nil\n\t\t})\n\t\treturn err\n\t}\n\n\t// Execute the transaction\n\terr = rdb.Watch(ctx, txf, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc getObject(ctx context.Context, rdb *redis.Client, key string) (*Object, error) {\n\tresult, err := rdb.Get(ctx, key).Bytes()\n\tif err != nil {\n\t\tif err == redis.Nil {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tvar obj Object\n\terr = json.Unmarshal(result, &obj)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &obj, nil\n}\n```\n\n### Explanation:\n\n1. **Initialization**: The Redis client is initialized with the appropriate connection details.\n2. **Object Definition**: An example object is defined with a value and a version.\n3. **Store Function**: The `storeIfNewerVersion` function checks if the current version in Redis is older than the new object's version. If it is, it updates the object in Redis.\n4. **Transaction**: The function uses a Redis transaction to ensure atomicity. It watches the key to prevent concurrent modifications.\n5. **Get Function**: The `getObject` function retrieves the current object from Redis and deserializes it from JSON.\n\nThis code ensures that the object is only stored in Redis if the version is newer than the version already present.",
              "label": {
                "is_preferred": true,
                "preference": "chosen",
                "response_type": "A"
              }
            }
          },
          {
            "answer": {
              "role": "assistant",
              "content": "To achieve this, you can use the `HSETNX` command in Redis, which sets the value of a hash field only if the field does not exist. In this case, we'll use a hash to store the object's value and version. We'll first check if the version in Redis is older than the given version, and if so, update the value. Here's the Golang code using the `go-redis` library:\n\n```go\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/go-redis/redis/v8\"\n)\n\ntype Object struct {\n\tValue   string\n\tVersion int\n}\n\nfunc StoreObjectIfNewer(ctx context.Context, redisClient *redis.Client, key string, obj Object) error {\n\t// Get the current version from Redis\n\tcurrentVersionStr, err := redisClient.HGet(ctx, key, \"version\").Result()\n\tif err != nil && err != redis.Nil {\n\t\treturn fmt.Errorf(\"error getting current version: %w\", err)\n\t}\n\n\t// If the key doesn't exist, set the value and version\n\tif err == redis.Nil {\n\t\treturn redisClient.HMSet(ctx, key, map[string]interface{}{\n\t\t\t\"value\":   obj.Value,\n\t\t\t\"version\": obj.Version,\n\t\t}).Err()\n\t}\n\n\t// Parse the current version\n\tcurrentVersion, err := strconv.Atoi(currentVersionStr)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error parsing current version: %w\", err)\n\t}\n\n\t// If the given version is newer, update the value\n\tif obj.Version > currentVersion {\n\t\treturn redisClient.HMSet(ctx, key, map[string]interface{}{\n\t\t\t\"value\":   obj.Value,\n\t\t\t\"version\": obj.Version,\n\t\t}).Err()\n\t}\n\n\treturn nil\n}\n```\n\nThis function first checks if the key exists in Redis. If it doesn't, it sets the value and version. If it does, it compares the given version with the current version in Redis and updates the value if the given version is newer.",
              "label": {
                "is_preferred": false,
                "preference": "rejected",
                "response_type": "B"
              }
            }
          }
        ],
        "preferred": "A",
        "domain": "code",
        "language": "go"
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-synthetic-text-to-sql-5098",
      "title": "Aggregate maintenance frequency by equipment type",
      "source": {
        "dataset": "gretelai/synthetic_text_to_sql",
        "url": "https://huggingface.co/datasets/gretelai/synthetic_text_to_sql/blob/740ab236e64503fba51be1101df7a1be83bf455d/synthetic_text_to_sql_train.snappy.parquet",
        "revision": "740ab236e64503fba51be1101df7a1be83bf455d",
        "split": "train",
        "row_id": "5098",
        "row_index_in_file": 1,
        "source_file": "synthetic_text_to_sql_train.snappy.parquet",
        "license": "Apache-2.0",
        "pool": "proposed",
        "row_sha256": "7049c382de9ccddc83722e2ac4c0bf3f61a9cd51117bbd3e467f6aad90e66be3",
        "attribution": "Source task and reference: gretelai/synthetic_text_to_sql. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/gretelai/synthetic_text_to_sql/blob/740ab236e64503fba51be1101df7a1be83bf455d/synthetic_text_to_sql_train.snappy.parquet"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "database_business_operations",
        "leaf": "database_business_operations.database_query_analysis",
        "confidence": "editorial_high",
        "reason": "Requires grouping and aggregating tabular records, not domain knowledge."
      },
      "language": "en",
      "prompt": "List all the unique equipment types and their corresponding total maintenance frequency from the equipment_maintenance table.",
      "context": [
        "CREATE TABLE equipment_maintenance (equipment_type VARCHAR(255), maintenance_frequency INT);"
      ],
      "environment_note": "Self-contained SQL schema. Editorial fixture tests use in-memory SQLite, not a live business database.",
      "rubric": {
        "markdown": "# Aggregate maintenance frequency by equipment type\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct grouping\n\n**ID:** c1\n\n**Criterion:** Return one row for every distinct equipment\\_type present, using SQL grouping semantics, without inventing a filter or dropping a type.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Correct total\n\n**ID:** c2\n\n**Criterion:** Compute SUM(maintenance\\_frequency) within each type. Equivalent SQL is accepted; COUNT of records does not compute total frequency.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\", \"Aggregation\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Executable query\n\n**ID:** c3\n\n**Criterion:** Produce executable SQL against the supplied schema. Aliases, CTEs, whitespace and unordered output may vary; no order was requested.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "sql": "SELECT equipment_type, SUM(maintenance_frequency) AS total_maintenance_frequency FROM equipment_maintenance GROUP BY equipment_type;",
          "sql_explanation": "This query groups the equipment_maintenance table by equipment_type and calculates the sum of maintenance_frequency for each group, then returns the equipment_type and the corresponding total_maintenance_frequency.",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Schema and source SQL are present.",
          "Two formulations produce the same results on nontrivial SQLite fixtures."
        ],
        "risks": [
          "Source context contains no example rows; fixtures are editorial.",
          "SQLite checks do not certify all SQL dialects."
        ],
        "must_hold": [
          "Group by equipment type.",
          "Sum the frequency values rather than count records."
        ],
        "may_vary": [
          "CTE versus direct aggregation.",
          "Column aliases and row order when not requested.",
          "Equivalent SQL syntax in a declared dialect."
        ],
        "proposed_changes": [
          "Judge result semantics on multiple fixtures, never exact SQL text."
        ]
      },
      "probes": [
        {
          "label": "Direct GROUP BY",
          "kind": "valid_alternative",
          "response": "SELECT equipment_type, SUM(maintenance_frequency) AS total_maintenance_frequency FROM equipment_maintenance GROUP BY equipment_type;",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "executed",
          "why": "Direct aggregation implements the requested sum.",
          "verification_detail": "Executed in SQLite on mixed duplicate/NULL/zero values and an empty table; matched expected result."
        },
        {
          "label": "CTE formulation",
          "kind": "valid_alternative",
          "response": "WITH totals AS (SELECT equipment_type AS kind, SUM(maintenance_frequency) AS frequency FROM equipment_maintenance GROUP BY equipment_type) SELECT kind AS equipment_type, frequency AS total_maintenance_frequency FROM totals;",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "executed",
          "why": "Changes query organization and aliases while preserving the relational result.",
          "verification_detail": "Executed on the same fixtures; equivalent results."
        },
        {
          "label": "Counts records instead of frequencies",
          "kind": "invalid_control",
          "response": "SELECT equipment_type, COUNT(*) AS total_maintenance_frequency FROM equipment_maintenance GROUP BY equipment_type;",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "executed",
          "why": "Two truck rows count as 2, while their maintenance frequencies total 7.",
          "verification_detail": "Executed on fixture; differs from expected totals."
        }
      ],
      "source_record": {
        "id": 5098,
        "domain": "defense industry",
        "domain_description": "Defense contract data, military equipment maintenance, threat intelligence metrics, and veteran employment stats.",
        "sql_complexity": "aggregation",
        "sql_complexity_description": "aggregation functions (COUNT, SUM, AVG, MIN, MAX, etc.), and HAVING clause",
        "sql_task_type": "analytics and reporting",
        "sql_task_type_description": "generating reports, dashboards, and analytical insights",
        "sql_prompt": "List all the unique equipment types and their corresponding total maintenance frequency from the equipment_maintenance table.",
        "sql_context": "CREATE TABLE equipment_maintenance (equipment_type VARCHAR(255), maintenance_frequency INT);",
        "sql": "SELECT equipment_type, SUM(maintenance_frequency) AS total_maintenance_frequency FROM equipment_maintenance GROUP BY equipment_type;",
        "sql_explanation": "This query groups the equipment_maintenance table by equipment_type and calculates the sum of maintenance_frequency for each group, then returns the equipment_type and the corresponding total_maintenance_frequency."
      },
      "verification": {
        "status": "executed",
        "method": "In-memory SQLite execution of two full SQL responses and an invalid control.",
        "checks": [
          {
            "name": "Direct GROUP BY",
            "passed": true,
            "detail": "Executed in SQLite on mixed duplicate/NULL/zero values and an empty table; matched expected result."
          },
          {
            "name": "CTE formulation",
            "passed": true,
            "detail": "Executed on the same fixtures; equivalent results."
          },
          {
            "name": "Counts records instead of frequencies — correctly rejected",
            "passed": true,
            "detail": "Executed on fixture; differs from expected totals."
          }
        ],
        "limitations": [
          "Executed checks compare results or stored table values, not model rubric scores.",
          "Fixtures are authored additions; finite cases do not prove every possible database behavior.",
          "No external database, triggers or transaction side effects were exercised."
        ]
      },
      "reference": {
        "answer": "SELECT equipment_type, SUM(maintenance_frequency) AS total_maintenance_frequency FROM equipment_maintenance GROUP BY equipment_type;",
        "solution": "This query groups the equipment_maintenance table by equipment_type and calculates the sum of maintenance_frequency for each group, then returns the equipment_type and the corresponding total_maintenance_frequency.",
        "tests": "Source supplies schema and SQL but no task-specific executable tests. The gallery adds SQLite fixtures.",
        "source_reference_verified": true,
        "warning": "Verified on the recorded finite SQLite fixtures, not every dialect or possible database."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-synthetic-text-to-sql-5109",
      "title": "Update only the specified carbon-offset records",
      "source": {
        "dataset": "gretelai/synthetic_text_to_sql",
        "url": "https://huggingface.co/datasets/gretelai/synthetic_text_to_sql/blob/740ab236e64503fba51be1101df7a1be83bf455d/synthetic_text_to_sql_train.snappy.parquet",
        "revision": "740ab236e64503fba51be1101df7a1be83bf455d",
        "split": "train",
        "row_id": "5109",
        "row_index_in_file": 12,
        "source_file": "synthetic_text_to_sql_train.snappy.parquet",
        "license": "Apache-2.0",
        "pool": "proposed",
        "row_sha256": "c3c7e74adb114dd541d4a6328308bdd733a119eb9a334e21a365c0b296c72d09",
        "attribution": "Source task and reference: gretelai/synthetic_text_to_sql. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/gretelai/synthetic_text_to_sql/blob/740ab236e64503fba51be1101df7a1be83bf455d/synthetic_text_to_sql_train.snappy.parquet"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "database_business_operations",
        "leaf": "database_business_operations.record_update_reconciliation",
        "confidence": "editorial_high",
        "reason": "Requires a scoped persistent record update with preservation of other records."
      },
      "language": "en",
      "prompt": "Update the end date of carbon offset initiative 'Initiative 1' in Australia to '2025-12-31'.",
      "context": [
        "CREATE TABLE carbon_offsets (initiative_id INT, initiative_name VARCHAR(255), country VARCHAR(255), start_date DATE, end_date DATE);"
      ],
      "environment_note": "Self-contained SQL schema without triggers or external effects. Verified only on an in-memory SQLite database.",
      "rubric": {
        "markdown": "# Update only the specified carbon-offset records\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Target records\n\n**ID:** c1\n\n**Criterion:** Set end\\_date to '2025-12-31' for rows matching both initiative\\_name = 'Initiative 1' and country = 'Australia'.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Preserve other data\n\n**ID:** c2\n\n**Criterion:** Leave other values unchanged. An equivalent scoped UPDATE is accepted; exact WHERE-clause order or formatting is irrelevant.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Data integrity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Executable statement\n\n**ID:** c3\n\n**Criterion:** Use executable SQL for the supplied schema without introducing unsupported schema assumptions.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "sql": "UPDATE carbon_offsets SET end_date = '2025-12-31' WHERE initiative_name = 'Initiative 1' AND country = 'Australia';",
          "sql_explanation": "This SQL query updates the end date of carbon offset initiative 'Initiative 1' in Australia to '2025-12-31' by modifying the end_date value in the carbon_offsets table where the initiative_name is 'Initiative 1' and country is 'Australia'.",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Source SQL agrees with the requested change.",
          "Fixture includes same name in a different country and duplicate target records."
        ],
        "risks": [
          "A broad CASE update can touch unchanged rows, affecting triggers or audit logs in environments not supplied here.",
          "Only table-state semantics were executed; production transaction effects were not tested."
        ],
        "must_hold": [
          "Match both the initiative name and country.",
          "Preserve all other values."
        ],
        "may_vary": [
          "Equivalent Boolean expressions and predicate order.",
          "Dialect-compatible syntax and transaction wrapper."
        ],
        "proposed_changes": [
          "Score target-state correctness separately from unnecessary row touches when a real environment includes triggers."
        ]
      },
      "probes": [
        {
          "label": "Scoped UPDATE",
          "kind": "valid_alternative",
          "response": "UPDATE carbon_offsets SET end_date = '2025-12-31' WHERE initiative_name = 'Initiative 1' AND country = 'Australia';",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "executed",
          "why": "Changes precisely the requested records.",
          "verification_detail": "Executed in SQLite against four records; exact final table matched."
        },
        {
          "label": "Conditional assignment alternative",
          "kind": "valid_alternative",
          "response": "UPDATE carbon_offsets SET end_date = CASE WHEN initiative_name = 'Initiative 1' AND country = 'Australia' THEN '2025-12-31' ELSE end_date END;",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "executed",
          "why": "Same table state on the supplied trigger-free schema. Would need stricter scoping if unchanged-row side effects mattered.",
          "verification_detail": "Executed in SQLite; all stored values match the scoped UPDATE. Trigger behavior not checked."
        },
        {
          "label": "Missing country constraint",
          "kind": "invalid_control",
          "response": "UPDATE carbon_offsets SET end_date = '2025-12-31' WHERE initiative_name = 'Initiative 1';",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "executed",
          "why": "Also changes the Canadian record with the same initiative name.",
          "verification_detail": "Executed in SQLite; final table differs on the Canadian record."
        }
      ],
      "source_record": {
        "id": 5109,
        "domain": "sustainable infrastructure",
        "domain_description": "Green building data, renewable energy infrastructure projects, carbon offset initiatives, and smart city technology adoption.",
        "sql_complexity": "basic SQL",
        "sql_complexity_description": "basic SQL with a simple select statement",
        "sql_task_type": "data manipulation",
        "sql_task_type_description": "inserting, updating, or deleting records",
        "sql_prompt": "Update the end date of carbon offset initiative 'Initiative 1' in Australia to '2025-12-31'.",
        "sql_context": "CREATE TABLE carbon_offsets (initiative_id INT, initiative_name VARCHAR(255), country VARCHAR(255), start_date DATE, end_date DATE);",
        "sql": "UPDATE carbon_offsets SET end_date = '2025-12-31' WHERE initiative_name = 'Initiative 1' AND country = 'Australia';",
        "sql_explanation": "This SQL query updates the end date of carbon offset initiative 'Initiative 1' in Australia to '2025-12-31' by modifying the end_date value in the carbon_offsets table where the initiative_name is 'Initiative 1' and country is 'Australia'."
      },
      "verification": {
        "status": "executed",
        "method": "In-memory SQLite execution of two full SQL responses and an invalid control.",
        "checks": [
          {
            "name": "Scoped UPDATE",
            "passed": true,
            "detail": "Executed in SQLite against four records; exact final table matched."
          },
          {
            "name": "Conditional assignment alternative",
            "passed": true,
            "detail": "Executed in SQLite; all stored values match the scoped UPDATE. Trigger behavior not checked."
          },
          {
            "name": "Missing country constraint — correctly rejected",
            "passed": true,
            "detail": "Executed in SQLite; final table differs on the Canadian record."
          }
        ],
        "limitations": [
          "Executed checks compare results or stored table values, not model rubric scores.",
          "Fixtures are authored additions; finite cases do not prove every possible database behavior.",
          "No external database, triggers or transaction side effects were exercised."
        ]
      },
      "reference": {
        "answer": "UPDATE carbon_offsets SET end_date = '2025-12-31' WHERE initiative_name = 'Initiative 1' AND country = 'Australia';",
        "solution": "This SQL query updates the end date of carbon offset initiative 'Initiative 1' in Australia to '2025-12-31' by modifying the end_date value in the carbon_offsets table where the initiative_name is 'Initiative 1' and country is 'Australia'.",
        "tests": "Source supplies schema and SQL but no task-specific executable tests. The gallery adds SQLite fixtures.",
        "source_reference_verified": true,
        "warning": "Verified on the recorded finite SQLite fixtures, not every dialect or possible database."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:mimo:s3k_1579_hospitality_fnb_zh_t3_rl_002",
      "title": "净销售复核：退款、员工餐与重复明细",
      "source": {
        "dataset": "XiaomiMiMo/MiMo-V2.6-RL-oss",
        "source_name": "mimo",
        "url": "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss",
        "revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "original_inventory_revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Original inventory verified Hugging Face commit",
        "split": "general/train",
        "row_id": "s3k_1579_hospitality_fnb_zh_t3_rl_002",
        "license": "Apache-2.0",
        "pool": "current",
        "source_content_sha256": "ade5348ace27b5b7891b2b70beaf8174a6d133d1bc82b7b4b1d355d7a530e616",
        "source_record_fingerprint": "6d98ca17927ced95ec2e881064b96f9c01bf7b60481484b4ba5e6de0788f11ae",
        "source_row_index": 10,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": false,
        "synthetic_claim_from_import": true,
        "attribution": "Source task and rubric: mimo; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.numerical_derivation",
        "confidence": "editorial_provisional",
        "reason": "The deliverable derives net sales and quantities under inclusion, refund, and deduplication rules."
      },
      "language": "zh",
      "prompt": "user: 运营在复核 Jamaica Restaurant 2021 年 5 月 1 日至 7 月 31 日的开胃菜表现，现有汇总数可能混入员工餐、作废小票和离线重传产生的重复明细。请在 Toast Web 菜单销售报表中独立核算 Buffalo Wings 与 Sunshine Tomato Soup 的有效销售表现，退款按净销售口径反映，不要把不属于正常营业销售的记录算进去。\n\n请在最终回复中分别给出两道菜的有效销售份数和净销售额，并保留营业日口径，列出实际产生有效销售的营业日及对应结果；同时说明你发现并排除了哪些类型的异常，以及这些异常若不处理会怎样影响销量。无需修改数据库或任何文件。",
      "context": [],
      "environment_note": "Original remote workspace and tools are required and are not included or executed. Response probes compare representations against source anchors only; no environment success is claimed.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### 是否正确算出 Buffalo Wings 在观察期内的有效销售份数与净销售额？\n\n**ID:** buffalo\\_effective\\_total\n\n**Criterion:** 是否正确算出 Buffalo Wings 在观察期内的有效销售份数与净销售额？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：Jamaica Restaurant - Main 在 2021-05-01 至 2021-07-31，按营业日关联小票与菜品明细，排除员工餐、作废/非正常营业小票和无效明细，并对同一来源事件的同菜品重复行仅保留明细标识最小的一行；退款计入净销售后，Buffalo Wings 有效销量为 450 份，净销售额为 5,325.00 美元（532,500 美分）。金额允许货币符号、千位分隔、小数位等自然格式差异，但数值含义必须一致。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Mathematical and statistical analysis\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 是否正确算出 Sunshine Tomato Soup 在观察期内的有效销售份数与净销售额？\n\n**ID:** soup\\_effective\\_total\n\n**Criterion:** 是否正确算出 Sunshine Tomato Soup 在观察期内的有效销售份数与净销售额？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：同一观察期和有效销售口径下，Sunshine Tomato Soup 有效销量为 43 份，净销售额为 260.00 美元（26,000 美分）。其正常销售毛额为 279.50 美元，退款金额为 19.50 美元，因此净额为 260.00 美元；不得把 2021-07-04 的作废销售并入。金额格式可自然变化。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Mathematical and statistical analysis\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 是否按营业日正确列出两道菜实际产生有效销售的日期及对应销量和净销售额？\n\n**ID:** business\\_day\\_breakdown\n\n**Criterion:** 是否按营业日正确列出两道菜实际产生有效销售的日期及对应销量和净销售额？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：有效营业日结果只有两组——2021-05-22 Buffalo Wings：450 份、净销售额 5,325.00 美元；2021-05-01 Sunshine Tomato Soup：43 份、净销售额 260.00 美元。观察期共 92 个自然日，但无需逐日列示零值；不得把 Buffalo Wings 的 2021-06-12 员工餐或 Sunshine Tomato Soup 的 2021-07-04 作废记录列作有效营业日销售。日期可用任意清晰自然语言格式。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Mathematical and statistical analysis\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 是否识别并正确解释 Buffalo Wings 的离线重传重复明细与员工餐排除，以及不排除时对销量的影响？\n\n**ID:** duplicate\\_and\\_staff\\_traps\n\n**Criterion:** 是否识别并正确解释 Buffalo Wings 的离线重传重复明细与员工餐排除，以及不排除时对销量的影响？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：Buffalo Wings 在 2021-05-22 有两条同一来源事件的明细，按最小明细标识保留主行后排除的重传重复行为 1 份、12.50 美元；2021-06-12 另有员工餐 5 份、62.50 美元，应排除。若直接累计原始销售数量，会在有效 450 份上多算 6 份而得到 456 份。无需写记录 ID，也无需在异常说明中重复列出异常日期。可以分别说明重复项使销量虚增 1 份、员工餐使销量虚增 5 份，也可以合并说明两者共虚增 6 份（450 份变为 456 份）；这些及其他语义等价表述均应通过。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Mathematical and statistical analysis\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 是否识别并正确解释 Sunshine Tomato Soup 的作废记录排除及其对销量的影响？\n\n**ID:** void\\_trap\n\n**Criterion:** 是否识别并正确解释 Sunshine Tomato Soup 的作废记录排除及其对销量的影响？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：Sunshine Tomato Soup 在 2021-07-04 有一笔已作废小票上的作废明细，数量 2 份、金额 13.00 美元，应完全排除；若直接累计原始数量，会把有效 43 份误报为 45 份。无需写记录 ID，也无需在异常说明中重复列出异常日期；只要明确作废销售不属于有效营业销售，并正确说明排除 2 份后有效销量为 43 份（不排除则为 45 份）即可。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Mathematical and statistical analysis\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "是否正确算出 Buffalo Wings 在观察期内的有效销售份数与净销售额？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：Jamaica Restaurant - Main 在 2021-05-01 至 2021-07-31，按营业日关联小票与菜品明细，排除员工餐、作废/非正常营业小票和无效明细，并对同一来源事件的同菜品重复行仅保留明细标识最小的一行；退款计入净销售后，Buffalo Wings 有效销量为 450 份，净销售额为 5,325.00 美元（532,500 美分）。金额允许货币符号、千位分隔、小数位等自然格式差异，但数值含义必须一致。"
              },
              "id": "buffalo_effective_total",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "是否正确算出 Sunshine Tomato Soup 在观察期内的有效销售份数与净销售额？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：同一观察期和有效销售口径下，Sunshine Tomato Soup 有效销量为 43 份，净销售额为 260.00 美元（26,000 美分）。其正常销售毛额为 279.50 美元，退款金额为 19.50 美元，因此净额为 260.00 美元；不得把 2021-07-04 的作废销售并入。金额格式可自然变化。"
              },
              "id": "soup_effective_total",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "是否按营业日正确列出两道菜实际产生有效销售的日期及对应销量和净销售额？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：有效营业日结果只有两组——2021-05-22 Buffalo Wings：450 份、净销售额 5,325.00 美元；2021-05-01 Sunshine Tomato Soup：43 份、净销售额 260.00 美元。观察期共 92 个自然日，但无需逐日列示零值；不得把 Buffalo Wings 的 2021-06-12 员工餐或 Sunshine Tomato Soup 的 2021-07-04 作废记录列作有效营业日销售。日期可用任意清晰自然语言格式。"
              },
              "id": "business_day_breakdown",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "是否识别并正确解释 Buffalo Wings 的离线重传重复明细与员工餐排除，以及不排除时对销量的影响？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：Buffalo Wings 在 2021-05-22 有两条同一来源事件的明细，按最小明细标识保留主行后排除的重传重复行为 1 份、12.50 美元；2021-06-12 另有员工餐 5 份、62.50 美元，应排除。若直接累计原始销售数量，会在有效 450 份上多算 6 份而得到 456 份。无需写记录 ID，也无需在异常说明中重复列出异常日期。可以分别说明重复项使销量虚增 1 份、员工餐使销量虚增 5 份，也可以合并说明两者共虚增 6 份（450 份变为 456 份）；这些及其他语义等价表述均应通过。"
              },
              "id": "duplicate_and_staff_traps",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "是否识别并正确解释 Sunshine Tomato Soup 的作废记录排除及其对销量的影响？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：Sunshine Tomato Soup 在 2021-07-04 有一笔已作废小票上的作废明细，数量 2 份、金额 13.00 美元，应完全排除；若直接累计原始数量，会把有效 43 份误报为 45 份。无需写记录 ID，也无需在异常说明中重复列出异常日期；只要明确作废销售不属于有效营业销售，并正确说明排除 2 份后有效销量为 43 份（不排除则为 45 份）即可。"
              },
              "id": "void_trap",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 5,
        "source_original_items": [
          {
            "files": [
              "answer.md"
            ],
            "id": "buffalo_effective_total",
            "method": "llm",
            "pass_anchor": "真实数据锚点：Jamaica Restaurant - Main 在 2021-05-01 至 2021-07-31，按营业日关联小票与菜品明细，排除员工餐、作废/非正常营业小票和无效明细，并对同一来源事件的同菜品重复行仅保留明细标识最小的一行；退款计入净销售后，Buffalo Wings 有效销量为 450 份，净销售额为 5,325.00 美元（532,500 美分）。金额允许货币符号、千位分隔、小数位等自然格式差异，但数值含义必须一致。",
            "question": "是否正确算出 Buffalo Wings 在观察期内的有效销售份数与净销售额？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "soup_effective_total",
            "method": "llm",
            "pass_anchor": "真实数据锚点：同一观察期和有效销售口径下，Sunshine Tomato Soup 有效销量为 43 份，净销售额为 260.00 美元（26,000 美分）。其正常销售毛额为 279.50 美元，退款金额为 19.50 美元，因此净额为 260.00 美元；不得把 2021-07-04 的作废销售并入。金额格式可自然变化。",
            "question": "是否正确算出 Sunshine Tomato Soup 在观察期内的有效销售份数与净销售额？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "business_day_breakdown",
            "method": "llm",
            "pass_anchor": "真实数据锚点：有效营业日结果只有两组——2021-05-22 Buffalo Wings：450 份、净销售额 5,325.00 美元；2021-05-01 Sunshine Tomato Soup：43 份、净销售额 260.00 美元。观察期共 92 个自然日，但无需逐日列示零值；不得把 Buffalo Wings 的 2021-06-12 员工餐或 Sunshine Tomato Soup 的 2021-07-04 作废记录列作有效营业日销售。日期可用任意清晰自然语言格式。",
            "question": "是否按营业日正确列出两道菜实际产生有效销售的日期及对应销量和净销售额？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "duplicate_and_staff_traps",
            "method": "llm",
            "pass_anchor": "真实数据锚点：Buffalo Wings 在 2021-05-22 有两条同一来源事件的明细，按最小明细标识保留主行后排除的重传重复行为 1 份、12.50 美元；2021-06-12 另有员工餐 5 份、62.50 美元，应排除。若直接累计原始销售数量，会在有效 450 份上多算 6 份而得到 456 份。无需写记录 ID，也无需在异常说明中重复列出异常日期。可以分别说明重复项使销量虚增 1 份、员工餐使销量虚增 5 份，也可以合并说明两者共虚增 6 份（450 份变为 456 份）；这些及其他语义等价表述均应通过。",
            "question": "是否识别并正确解释 Buffalo Wings 的离线重传重复明细与员工餐排除，以及不排除时对销量的影响？",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "void_trap",
            "method": "llm",
            "pass_anchor": "真实数据锚点：Sunshine Tomato Soup 在 2021-07-04 有一笔已作废小票上的作废明细，数量 2 份、金额 13.00 美元，应完全排除；若直接累计原始数量，会把有效 43 份误报为 45 份。无需写记录 ID，也无需在异常说明中重复列出异常日期；只要明确作废销售不属于有效营业销售，并正确说明排除 2 份后有效销量为 43 份（不排除则为 45 份）即可。",
            "question": "是否识别并正确解释 Sunshine Tomato Soup 的作废记录排除及其对销量的影响？",
            "tier": "important"
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Weighted sum divided by all criterion weights; missing source weights are 1. Tiers are metadata, not implicit gates.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Anchors allow equivalent currency/date formats and combined or separate anomaly explanations.",
          "Tests aggregation and specific traps, not a single SQL query or reasoning sequence."
        ],
        "risks": [
          "Toast records are not supplied here; arithmetic consistency is checkable but source truth is not.",
          "Adapter default points do not establish the source's intended weighting."
        ],
        "must_hold": [
          "Buffalo Wings 450 portions and USD 5,325; soup 43 portions and USD 260, subject to source verification.",
          "Respect business days, exclude duplicate/staff/void records, and count refunds in net revenue."
        ],
        "may_vary": [
          "Amounts in dollars or cents with explicit units; dates in any unambiguous format.",
          "SQL, spreadsheet, or other correct aggregation; list anomalies separately or combine their quantity effects."
        ],
        "proposed_changes": [
          "Add numeric equivalence checks with explicit units and retain the source's wording flexibility.",
          "Grade the underlying read-only calculation independently of its final prose."
        ]
      },
      "probes": [
        {
          "label": "Source-agreement candidate A: dollars and table",
          "kind": "valid_alternative",
          "response": "按 2021-05-01 至 2021-07-31 的营业日口径，排除员工餐、作废或非正常营业小票及无效明细；同一来源事件的同菜品重复明细只保留最小明细标识，退款扣减销售额。\n\n| 有效营业日 | 菜品 | 有效份数 | 净销售额（美元） |\n|---|---|---:|---:|\n| 2021-05-22 | Buffalo Wings | 450 | 5,325.00 |\n| 2021-05-01 | Sunshine Tomato Soup | 43 | 260.00 |\n\n上述也是两道菜各自在全期的合计。Buffalo Wings 排除了离线重传重复的 1 份、12.50 美元及员工餐 5 份、62.50 美元；若直接累计会多算 6 份，误报 456 份。汤的正常毛额 279.50 美元扣除退款 19.50 美元后为 260.00 美元；另排除作废销售 2 份、13.00 美元，否则会误报 45 份。92 个自然日中只列有有效销售的营业日，不把员工餐或作废日期列为销售日。",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Source-anchor agreement plus exact arithmetic checks in builder; Toast data not executed.",
          "why": "Uses a table and dollar values with explicit inclusion rules."
        },
        {
          "label": "Source-agreement candidate B: cents and prose",
          "kind": "valid_alternative",
          "response": "核算只包含正常营业的有效记录，以营业日归属，剔除员工餐、作废及无效明细，并对同一来源事件的同菜品重复行保留最小明细标识。金额以下均为美分，退款已经从收入中扣除。\n\nSunshine Tomato Soup 在 2021年5月1日有效销售 43 份，净额 26,000 美分；这是全观察期该菜的合计，毛额 27,950 减退款 1,950 得到净额。7月4日作废的 2 份、1,300 美分不计入，否则份数会从 43 虚增至 45。\n\nBuffalo Wings 在 2021年5月22日有效销售 450 份，净额 532,500 美分，也是全期合计。排除重传重复的 1 份、1,250 美分和6月12日员工餐的 5 份、6,250 美分；二者若混入，共多算 6 份成为 456 份。全期为5月1日至7月31日，只有以上两组有效营业日结果，其余日期无需逐日列零。",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Source-anchor agreement plus exact arithmetic checks in builder; Toast data not executed.",
          "why": "Changes order, currency units, dates and presentation while retaining the same mathematical result."
        },
        {
          "label": "Invalid: ignore refunds and duplicates",
          "kind": "invalid_control",
          "response": "Buffalo Wings 有效销量 456 份；Sunshine Tomato Soup 有效销量 45 份、净销售额 279.50 美元。",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Counts excluded rows and treats soup gross revenue as net revenue."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-numinamath-1.5-112",
      "title": "Prove a prime-indexed expression is never a square",
      "source": {
        "dataset": "AI-MO/NuminaMath-1.5",
        "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-1.5/blob/1b05109f9e5c1ad06c0663519502416c30b300f8/data/train-00000-of-00003.parquet",
        "revision": "1b05109f9e5c1ad06c0663519502416c30b300f8",
        "split": "train",
        "row_id": "112",
        "row_index_in_file": 112,
        "source_file": "data/train-00000-of-00003.parquet",
        "license": "Apache-2.0 (dataset); upstream olympiad attribution retained in source row",
        "pool": "proposed",
        "row_sha256": "ac1f4ebd6d400a91eecb947e2b261ff05346fae4dd08e5886b3a0b7cba3ef5ce",
        "attribution": "Source task and reference: AI-MO/NuminaMath-1.5. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-1.5/blob/1b05109f9e5c1ad06c0663519502416c30b300f8/data/train-00000-of-00003.parquet"
          }
        ]
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.proof_symbolic_reasoning",
        "confidence": "editorial_high",
        "reason": "Requires a universal arithmetic proof, not an answer lookup or finite search."
      },
      "language": "en",
      "prompt": "\nNT5 Let $p$ be a prime number. Show that $7 p+3^{p}-4$ is not a perfect square.\n\n",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Prove a prime-indexed expression is never a square\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Universal coverage\n\n**ID:** c1\n\n**Criterion:** Cover all prime p, including small exceptional primes when the chosen reasoning needs them.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Completeness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Valid contradiction or exclusion\n\n**ID:** c2\n\n**Criterion:** Supply a logically valid argument excluding perfect squares for every remaining prime. Any correct proof is accepted; no specific theorem or residue split is mandatory.\n\n**Points:** +6\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Proof\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Accurate reasoning\n\n**ID:** c3\n\n**Criterion:** Use arithmetic and theorem assumptions correctly. Numerical tests may illustrate but cannot establish the universal claim.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "answer": "proof",
          "solution": "## Solution\n\nAssume that for a prime number $p$ greater than $3, m=7 p+3^{p}-4$ is a perfect square. Let $m=n^{2}$ for some $n \\in \\mathbb{Z}$. By Fermat's Little Theorem,\n\n$$\nm=7 p+3^{p}-4 \\equiv 3-4 \\equiv-1 \\quad(\\bmod p)\n$$\n\nIf $p=4 k+3, k \\in \\mathbb{Z}$, then again by Fermat's Little Theorem\n\n$$\n-1 \\equiv m^{2 k+1} \\equiv n^{4 k+2} \\equiv n^{p-1} \\equiv 1 \\quad(\\bmod p), \\text { but } p>3\n$$\n\na contradiction. So $p \\equiv 1(\\bmod 4)$.\n\nTherefore $m=7 p+3^{p}-4 \\equiv 3-1 \\equiv 2(\\bmod 4)$. But this is a contradiction since 2 is not perfect square in $(\\bmod 4)$. For $p=2$ we have $m=19$ and for $p=3$ we have $m=44$.\n\n",
          "upstream_source": "olympiads",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Source includes a complete proof instead of only the answer token proof."
        ],
        "risks": [
          "A grader could wrongly require Fermat by matching the supplied solution.",
          "No formal proof assistant validation was run."
        ],
        "must_hold": [
          "Every prime is covered.",
          "Proof is logically valid and applies universally."
        ],
        "may_vary": [
          "Equivalent residue arguments, quadratic-residue theory, or another complete proof.",
          "Separate or integrated treatment of p=2 and p=3."
        ],
        "proposed_changes": [
          "Make universal validity the criterion; the particular modular method is illustrative."
        ]
      },
      "probes": [
        {
          "label": "Case-split proof",
          "kind": "valid_alternative",
          "response": "For p=2 and p=3, the expression is 19 and 44, neither a square. Let p>3 be prime. If p=1 mod 4, then 7p+3^p-4=2 mod 4, impossible for a square. If p=3 mod 4, suppose the expression is n^2. Fermat gives 3^p=3 mod p, so n^2=-1 mod p and p does not divide n. Raising to (p-1)/2, which is odd, yields n^(p-1)=-1 mod p, contradicting Fermat n^(p-1)=1 mod p. Both possible odd-prime cases are excluded.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Covers small primes and both odd-prime residue classes.",
          "verification_detail": "Editorial mathematical review of the modular contradictions."
        },
        {
          "label": "Alternative proof route to test",
          "kind": "valid_alternative",
          "response": "Use the fact that -1 is a quadratic nonresidue for primes congruent to 3 modulo 4, and exclude primes congruent to 1 modulo 4 directly modulo 4; handle the small primes.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "An equivalent route is legitimate, but this is a review sketch rather than a submitted full proof.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Finite checking only",
          "kind": "invalid_control",
          "response": "I checked all primes below 1000 and none gives a square. Therefore this is true for every prime.",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "Finite checking does not prove the unbounded universal statement.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "problem": "\nNT5 Let $p$ be a prime number. Show that $7 p+3^{p}-4$ is not a perfect square.\n\n",
        "solution": "## Solution\n\nAssume that for a prime number $p$ greater than $3, m=7 p+3^{p}-4$ is a perfect square. Let $m=n^{2}$ for some $n \\in \\mathbb{Z}$. By Fermat's Little Theorem,\n\n$$\nm=7 p+3^{p}-4 \\equiv 3-4 \\equiv-1 \\quad(\\bmod p)\n$$\n\nIf $p=4 k+3, k \\in \\mathbb{Z}$, then again by Fermat's Little Theorem\n\n$$\n-1 \\equiv m^{2 k+1} \\equiv n^{4 k+2} \\equiv n^{p-1} \\equiv 1 \\quad(\\bmod p), \\text { but } p>3\n$$\n\na contradiction. So $p \\equiv 1(\\bmod 4)$.\n\nTherefore $m=7 p+3^{p}-4 \\equiv 3-1 \\equiv 2(\\bmod 4)$. But this is a contradiction since 2 is not perfect square in $(\\bmod 4)$. For $p=2$ we have $m=19$ and for $p=3$ we have $m=44$.\n\n",
        "answer": "proof",
        "problem_type": "Number Theory",
        "question_type": "proof",
        "problem_is_valid": "Yes",
        "solution_is_valid": "Yes",
        "source": "olympiads",
        "synthetic": false
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "reference": {
        "answer": "proof",
        "solution": "## Solution\n\nAssume that for a prime number $p$ greater than $3, m=7 p+3^{p}-4$ is a perfect square. Let $m=n^{2}$ for some $n \\in \\mathbb{Z}$. By Fermat's Little Theorem,\n\n$$\nm=7 p+3^{p}-4 \\equiv 3-4 \\equiv-1 \\quad(\\bmod p)\n$$\n\nIf $p=4 k+3, k \\in \\mathbb{Z}$, then again by Fermat's Little Theorem\n\n$$\n-1 \\equiv m^{2 k+1} \\equiv n^{4 k+2} \\equiv n^{p-1} \\equiv 1 \\quad(\\bmod p), \\text { but } p>3\n$$\n\na contradiction. So $p \\equiv 1(\\bmod 4)$.\n\nTherefore $m=7 p+3^{p}-4 \\equiv 3-1 \\equiv 2(\\bmod 4)$. But this is a contradiction since 2 is not perfect square in $(\\bmod 4)$. For $p=2$ we have $m=19$ and for $p=3$ we have $m=44$.\n\n",
        "tests": [],
        "warning": "The proof is reviewed editorially. No formal proof assistant validation was run."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-numinamath-1.5-69",
      "title": "Maximum of a difference of square roots",
      "source": {
        "dataset": "AI-MO/NuminaMath-1.5",
        "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-1.5/blob/1b05109f9e5c1ad06c0663519502416c30b300f8/data/train-00000-of-00003.parquet",
        "revision": "1b05109f9e5c1ad06c0663519502416c30b300f8",
        "split": "train",
        "row_id": "69",
        "row_index_in_file": 69,
        "source_file": "data/train-00000-of-00003.parquet",
        "license": "Apache-2.0 (dataset); upstream olympiad attribution retained in source row",
        "pool": "proposed",
        "row_sha256": "4f9737a0ac623e5df5b40a4a30c243f658da4dc79742edd658018ca42f3c0afe",
        "attribution": "Source task and reference: AI-MO/NuminaMath-1.5. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-1.5/blob/1b05109f9e5c1ad06c0663519502416c30b300f8/data/train-00000-of-00003.parquet"
          }
        ]
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.optimization_numerical_modeling",
        "confidence": "editorial_high",
        "reason": "Requires proving a global bound and an attaining input over the real numbers."
      },
      "language": "en",
      "prompt": "\nA2. Find the largest possible value of the expression $\\left|\\sqrt{x^{2}+4 x+8}-\\sqrt{x^{2}+8 x+17}\\right|$ where $x$ is a real number.\n",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Maximum of a difference of square roots\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct maximum\n\n**ID:** c1\n\n**Criterion:** Identify sqrt(5) as the maximum, accepting equivalent exact expressions.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Global bound\n\n**ID:** c2\n\n**Criterion:** Give valid reasoning that the expression is at most sqrt(5) for every real x. Geometry, algebra, calculus or another complete argument are equally acceptable.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Proof\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Attainment\n\n**ID:** c3\n\n**Criterion:** Demonstrate that the bound is reached, for example at x=-6; a limit or numerical search alone is insufficient.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Completeness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "answer": "\\sqrt{5}",
          "solution": "\nSolution. We observe that\n\n$$\n\\left|\\sqrt{x^{2}+4 x+8}-\\sqrt{x^{2}+8 x+17}\\right|=\\left|\\sqrt{(x-(-2))^{2}+(0-2)^{2}}-\\sqrt{\\left.(x-(-4))^{2}+(0-1)^{2}\\right)}\\right|\n$$\n\nis the absolute difference of the distances from the point $P(x, 0)$ in the $x y$-plane to the points $A(-2,2)$ and $B(-4,1)$.\n\nBy the Triangle Inequality, $|P A-P B| \\leq|A B|$ and the equality occurs exactly when $P$ lies on the line passing through $A$ and $B$, but not between them.\n\nIf $P, A, B$ are collinear, then $(x-(-2)) /(0-2)=((-4)-(-2)) /(1-2)$. This gives $x=-6$, and as $-6<-4<-2$,\n\n$$\n\\left|\\sqrt{(-6)^{2}+4(-6)+8}-\\sqrt{(-6)^{2}+8(-6)+17}\\right|=|\\sqrt{20}-\\sqrt{5}|=\\sqrt{5}\n$$\n\nis the largest possible value of the expression.\n",
          "problem_type": "Algebra",
          "upstream_source": "olympiads",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Reference solution is a complete geometric argument.",
          "An independently drafted algebraic proof reaches the same bound."
        ],
        "risks": [
          "Finite integer checks are regression evidence only, not a proof of the real-domain inequality."
        ],
        "must_hold": [
          "A global upper bound.",
          "A real input attaining the maximum."
        ],
        "may_vary": [
          "Geometric reverse-triangle proof versus algebraic inequality versus sound calculus.",
          "Order of proof, notation and exact expression of sqrt(5)."
        ],
        "proposed_changes": [
          "Grade proof obligations rather than reproducing the reference geometry."
        ]
      },
      "probes": [
        {
          "label": "Geometric proof",
          "kind": "valid_alternative",
          "response": "Let P=(x,0), A=(-2,2), B=(-4,1). The expression is |PA-PB|. By the reverse triangle inequality it is at most AB=sqrt(5). At x=-6, the two distances are sqrt(20) and sqrt(5), so the difference is sqrt(5). Thus the largest value is sqrt(5), attained at x=-6.",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Uses a distance interpretation and exhibits equality.",
          "verification_detail": "Editorial proof review: reverse triangle inequality and explicit x=-6 evaluation."
        },
        {
          "label": "Algebraic proof",
          "kind": "valid_alternative",
          "response": "Put A=x^2+4x+8, B=x^2+8x+17 and C=x^2+6x+10=(x+3)^2+1>0. Expanding gives AB-C^2=(x+6)^2>=0, so sqrt(AB)>=C. Hence (sqrt(A)-sqrt(B))^2=A+B-2sqrt(AB)<=A+B-2C=5. Equality holds at x=-6. Therefore the maximum absolute difference is sqrt(5).",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Uses an exact polynomial identity instead of geometry.",
          "verification_detail": "Identity expanded and reviewed symbolically; integer checks x=-20..20 executed as supplementary regression checks."
        },
        {
          "label": "Confuses limit with maximum",
          "kind": "invalid_control",
          "response": "The expression approaches 2 as x tends to infinity, so its maximum is 2.",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "At x=-6 the value is sqrt(5)>2, disproving the claimed maximum.",
          "verification_detail": "Exact evaluation at x=-6 gives |sqrt(20)-sqrt(5)|=sqrt(5)."
        }
      ],
      "source_record": {
        "problem": "\nA2. Find the largest possible value of the expression $\\left|\\sqrt{x^{2}+4 x+8}-\\sqrt{x^{2}+8 x+17}\\right|$ where $x$ is a real number.\n",
        "solution": "\nSolution. We observe that\n\n$$\n\\left|\\sqrt{x^{2}+4 x+8}-\\sqrt{x^{2}+8 x+17}\\right|=\\left|\\sqrt{(x-(-2))^{2}+(0-2)^{2}}-\\sqrt{\\left.(x-(-4))^{2}+(0-1)^{2}\\right)}\\right|\n$$\n\nis the absolute difference of the distances from the point $P(x, 0)$ in the $x y$-plane to the points $A(-2,2)$ and $B(-4,1)$.\n\nBy the Triangle Inequality, $|P A-P B| \\leq|A B|$ and the equality occurs exactly when $P$ lies on the line passing through $A$ and $B$, but not between them.\n\nIf $P, A, B$ are collinear, then $(x-(-2)) /(0-2)=((-4)-(-2)) /(1-2)$. This gives $x=-6$, and as $-6<-4<-2$,\n\n$$\n\\left|\\sqrt{(-6)^{2}+4(-6)+8}-\\sqrt{(-6)^{2}+8(-6)+17}\\right|=|\\sqrt{20}-\\sqrt{5}|=\\sqrt{5}\n$$\n\nis the largest possible value of the expression.\n",
        "answer": "\\sqrt{5}",
        "problem_type": "Algebra",
        "question_type": "math-word-problem",
        "problem_is_valid": "Yes",
        "solution_is_valid": "Yes",
        "source": "olympiads",
        "synthetic": false
      },
      "verification": {
        "status": "executed",
        "method": "Integer algebra regression checks only; complete proofs remain editorially reviewed.",
        "checks": [
          {
            "name": "Polynomial identity regression",
            "passed": true,
            "detail": "For integer x=-20..20, checked AB-C^2=(x+6)^2 and A+B-2C=5."
          },
          {
            "name": "Attainment input arithmetic",
            "passed": true,
            "detail": "At x=-6, the two radicands evaluate to 20 and 5. The square-root conclusion is reviewed mathematically."
          }
        ],
        "limitations": [
          "Finite integer checks cannot certify the inequality for all real x.",
          "Both displayed full proofs were reviewed editorially, not executed or formally verified.",
          "No model rubric-judge run was performed."
        ]
      },
      "reference": {
        "answer": "\\sqrt{5}",
        "solution": "\nSolution. We observe that\n\n$$\n\\left|\\sqrt{x^{2}+4 x+8}-\\sqrt{x^{2}+8 x+17}\\right|=\\left|\\sqrt{(x-(-2))^{2}+(0-2)^{2}}-\\sqrt{\\left.(x-(-4))^{2}+(0-1)^{2}\\right)}\\right|\n$$\n\nis the absolute difference of the distances from the point $P(x, 0)$ in the $x y$-plane to the points $A(-2,2)$ and $B(-4,1)$.\n\nBy the Triangle Inequality, $|P A-P B| \\leq|A B|$ and the equality occurs exactly when $P$ lies on the line passing through $A$ and $B$, but not between them.\n\nIf $P, A, B$ are collinear, then $(x-(-2)) /(0-2)=((-4)-(-2)) /(1-2)$. This gives $x=-6$, and as $-6<-4<-2$,\n\n$$\n\\left|\\sqrt{(-6)^{2}+4(-6)+8}-\\sqrt{(-6)^{2}+8(-6)+17}\\right|=|\\sqrt{20}-\\sqrt{5}|=\\sqrt{5}\n$$\n\nis the largest possible value of the expression.\n",
        "tests": [],
        "warning": "The proof is reviewed editorially. No formal proof assistant validation was run."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "verified:gsm8k:train:0",
      "title": "Two months of clip sales",
      "source": {
        "dataset": "openai/gsm8k",
        "revision": "740312add88f781978c0658806c59bc2815b9866",
        "license": "MIT",
        "url": "https://huggingface.co/datasets/openai/gsm8k/blob/740312add88f781978c0658806c59bc2815b9866/main/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "0",
        "pool": "proposed",
        "file_sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d",
        "row_sha256": "ca002c0752ef5655e59bd767e52f501c591d0a3e5e2f8fffd23204716d4ed544",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Karl Cobbe et al., Training Verifiers to Solve Math Word Problems (2021), OpenAI GSM8K. Original train questions and solutions; editorial rubrics and response probes added."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.numerical_derivation",
        "confidence": "high",
        "reason": "An exact arithmetic result with multiple valid derivations."
      },
      "language": "en",
      "prompt": "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?",
      "context": [],
      "environment_note": "Self-contained arithmetic; no external environment needed.",
      "rubric": {
        "markdown": "# Two months of clip sales\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct requested quantity\n\n**ID:** answer\n\n**Criterion:** Gives the requested quantity as 72, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** The final answer is mathematically equivalent to the stated quantity.\n- **0:** The answer is incorrect, absent, or contradicted by another final answer.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_answer\", \"arithmetic\\_check\"\\]\n\n### Valid reasoning when supplied\n\n**ID:** faithfulness\n\n**Criterion:** Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Explanation, if present, is valid; a correct answer without a derivation also passes.\n- **0:** The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"reviewed\\_alternative\\_solutions\"\\]\n",
        "raw": {
          "title": "Two months of clip sales",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "answer",
              "criterion": "Gives the requested quantity as 72, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "The final answer is mathematically equivalent to the stated quantity.",
                "0": "The answer is incorrect, absent, or contradicted by another final answer."
              },
              "evidence_refs": [
                "source_answer",
                "arithmetic_check"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            },
            {
              "id": "faithfulness",
              "criterion": "Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.",
              "weight": 2.0,
              "grading_anchors": {
                "1": "Explanation, if present, is valid; a correct answer without a derivation also passes.",
                "0": "The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails."
              },
              "evidence_refs": [
                "task",
                "reviewed_alternative_solutions"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "answer": {
            "title": "Correct requested quantity",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "faithfulness": {
            "title": "Valid reasoning when supplied",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "72",
        "solution": "Natalia sold 48/2 = <<48/2=24>>24 clips in May.\nNatalia sold 48+24 = <<48+24=72>>72 clips altogether in April and May.\n#### 72",
        "tests": [
          "48 + 48 / 2 == 72",
          "3 * 48 / 2 == 72",
          "72 == 72"
        ]
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "A reference answer independently cross-checked by arithmetic.",
          "Accepts concise answers and different valid derivations."
        ],
        "risks": [
          "Matching a final number cannot certify arbitrary mathematical reasoning.",
          "These are deliberately simple conversion demonstrations, not evidence of advanced-math coverage."
        ],
        "must_hold": [
          "The requested quantity is 72.",
          "Any volunteered reasoning must be sound."
        ],
        "may_vary": [
          "Algebra, arithmetic, proportions, or a concise answer.",
          "Wording, notation, layout, and order of valid steps."
        ],
        "proposed_changes": [
          "Do not turn every reference-solution step into a mandatory criterion.",
          "Verify explanations independently when extending this to proofs."
        ]
      },
      "probes": [
        {
          "label": "Add the monthly amounts",
          "kind": "valid_alternative",
          "response": "May sales are half of 48, or 24 clips. The two-month total is 48 + 24 = 72 clips.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 72. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 72.0,
            "reference": 72,
            "passed": true
          }
        },
        {
          "label": "Scale the April amount",
          "kind": "valid_alternative",
          "response": "The combined sales are 1.5 times April's sales: 1.5 × 48 = 72 clips.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 72. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 72.0,
            "reference": 72,
            "passed": true
          }
        },
        {
          "label": "Concise answer",
          "kind": "valid_alternative",
          "response": "72 clips.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 72. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 72,
            "reference": 72,
            "passed": true
          }
        },
        {
          "label": "Incorrect reasoning control",
          "kind": "invalid_control",
          "response": "48 − 24 = 24 clips altogether.",
          "is_full_response": true,
          "expected": "Reject: incorrect quantity and reasoning.",
          "verification": "executed",
          "why": "The displayed calculation produces a number unequal to the verified source answer.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 24,
            "reference": 72,
            "passed": false
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Exact arithmetic checks plus editorial review of displayed responses; no model rubric-judge run.",
        "checks": [
          {
            "name": "Add the monthly amounts",
            "passed": true,
            "detail": "Authored arithmetic expression 48 + 48 / 2 evaluates to 72; prose reviewed separately."
          },
          {
            "name": "Scale the April amount",
            "passed": true,
            "detail": "Authored arithmetic expression 3 * 48 / 2 evaluates to 72; prose reviewed separately."
          },
          {
            "name": "Concise answer",
            "passed": true,
            "detail": "Authored arithmetic expression 72 evaluates to 72; prose reviewed separately."
          },
          {
            "name": "Wrong-answer control is rejected",
            "passed": true,
            "detail": "24 differs from the reference answer 72."
          }
        ],
        "limitations": [
          "Only the numeric check is executable. Natural-language reasoning and the draft rubric were reviewed editorially.",
          "The three correct responses are constructed coverage probes, not sampled student completions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "verified:gsm8k:train:1",
      "title": "Babysitting pay for a partial hour",
      "source": {
        "dataset": "openai/gsm8k",
        "revision": "740312add88f781978c0658806c59bc2815b9866",
        "license": "MIT",
        "url": "https://huggingface.co/datasets/openai/gsm8k/blob/740312add88f781978c0658806c59bc2815b9866/main/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "1",
        "pool": "proposed",
        "file_sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d",
        "row_sha256": "284293a3972590b15345932efab6576d4925079493994470268268a0292844b2",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Karl Cobbe et al., Training Verifiers to Solve Math Word Problems (2021), OpenAI GSM8K. Original train questions and solutions; editorial rubrics and response probes added."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.numerical_derivation",
        "confidence": "high",
        "reason": "An exact arithmetic result with multiple valid derivations."
      },
      "language": "en",
      "prompt": "Weng earns $12 an hour for babysitting. Yesterday, she just did 50 minutes of babysitting. How much did she earn?",
      "context": [],
      "environment_note": "Self-contained arithmetic; no external environment needed.",
      "rubric": {
        "markdown": "# Babysitting pay for a partial hour\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct requested quantity\n\n**ID:** answer\n\n**Criterion:** Gives the requested quantity as 10, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** The final answer is mathematically equivalent to the stated quantity.\n- **0:** The answer is incorrect, absent, or contradicted by another final answer.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_answer\", \"arithmetic\\_check\"\\]\n\n### Valid reasoning when supplied\n\n**ID:** faithfulness\n\n**Criterion:** Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Explanation, if present, is valid; a correct answer without a derivation also passes.\n- **0:** The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"reviewed\\_alternative\\_solutions\"\\]\n",
        "raw": {
          "title": "Babysitting pay for a partial hour",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "answer",
              "criterion": "Gives the requested quantity as 10, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "The final answer is mathematically equivalent to the stated quantity.",
                "0": "The answer is incorrect, absent, or contradicted by another final answer."
              },
              "evidence_refs": [
                "source_answer",
                "arithmetic_check"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            },
            {
              "id": "faithfulness",
              "criterion": "Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.",
              "weight": 2.0,
              "grading_anchors": {
                "1": "Explanation, if present, is valid; a correct answer without a derivation also passes.",
                "0": "The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails."
              },
              "evidence_refs": [
                "task",
                "reviewed_alternative_solutions"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "answer": {
            "title": "Correct requested quantity",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "faithfulness": {
            "title": "Valid reasoning when supplied",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "10",
        "solution": "Weng earns 12/60 = $<<12/60=0.2>>0.2 per minute.\nWorking 50 minutes, she earned 0.2 x 50 = $<<0.2*50=10>>10.\n#### 10",
        "tests": [
          "12 * 50 / 60 == 10",
          "(12 / 6) * 5 == 10",
          "10 == 10"
        ]
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "A reference answer independently cross-checked by arithmetic.",
          "Accepts concise answers and different valid derivations."
        ],
        "risks": [
          "Matching a final number cannot certify arbitrary mathematical reasoning.",
          "These are deliberately simple conversion demonstrations, not evidence of advanced-math coverage."
        ],
        "must_hold": [
          "The requested quantity is 10.",
          "Any volunteered reasoning must be sound."
        ],
        "may_vary": [
          "Algebra, arithmetic, proportions, or a concise answer.",
          "Wording, notation, layout, and order of valid steps."
        ],
        "proposed_changes": [
          "Do not turn every reference-solution step into a mandatory criterion.",
          "Verify explanations independently when extending this to proofs."
        ]
      },
      "probes": [
        {
          "label": "Convert minutes to hours",
          "kind": "valid_alternative",
          "response": "50 minutes is 5/6 of an hour, so the pay is $12 × 5/6 = $10.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 10. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 10.0,
            "reference": 10,
            "passed": true
          }
        },
        {
          "label": "Use ten-minute blocks",
          "kind": "valid_alternative",
          "response": "At $12 an hour, each 10-minute block earns $2. Five such blocks earn $10.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 10. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 10.0,
            "reference": 10,
            "passed": true
          }
        },
        {
          "label": "Concise answer",
          "kind": "valid_alternative",
          "response": "$10.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 10. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 10,
            "reference": 10,
            "passed": true
          }
        },
        {
          "label": "Incorrect reasoning control",
          "kind": "invalid_control",
          "response": "50 minutes is 0.50 hours, so the pay is $6.",
          "is_full_response": true,
          "expected": "Reject: incorrect quantity and reasoning.",
          "verification": "executed",
          "why": "The displayed calculation produces a number unequal to the verified source answer.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 6,
            "reference": 10,
            "passed": false
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Exact arithmetic checks plus editorial review of displayed responses; no model rubric-judge run.",
        "checks": [
          {
            "name": "Convert minutes to hours",
            "passed": true,
            "detail": "Authored arithmetic expression 12 * 50 / 60 evaluates to 10; prose reviewed separately."
          },
          {
            "name": "Use ten-minute blocks",
            "passed": true,
            "detail": "Authored arithmetic expression (12 / 6) * 5 evaluates to 10; prose reviewed separately."
          },
          {
            "name": "Concise answer",
            "passed": true,
            "detail": "Authored arithmetic expression 10 evaluates to 10; prose reviewed separately."
          },
          {
            "name": "Wrong-answer control is rejected",
            "passed": true,
            "detail": "6 differs from the reference answer 10."
          }
        ],
        "limitations": [
          "Only the numeric check is executable. Natural-language reasoning and the draft rubric were reviewed editorially.",
          "The three correct responses are constructed coverage probes, not sampled student completions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "verified:gsm8k:train:10",
      "title": "Recover the first ship's passenger count",
      "source": {
        "dataset": "openai/gsm8k",
        "revision": "740312add88f781978c0658806c59bc2815b9866",
        "license": "MIT",
        "url": "https://huggingface.co/datasets/openai/gsm8k/blob/740312add88f781978c0658806c59bc2815b9866/main/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "10",
        "pool": "proposed",
        "file_sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d",
        "row_sha256": "0de7380204a44869b4bf1db06247fc9aac419f2db3ab7146893fba9dcad554c2",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Karl Cobbe et al., Training Verifiers to Solve Math Word Problems (2021), OpenAI GSM8K. Original train questions and solutions; editorial rubrics and response probes added."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "mathematical_statistical_analysis",
        "leaf": "mathematical_statistical_analysis.numerical_derivation",
        "confidence": "high",
        "reason": "An exact arithmetic result with multiple valid derivations."
      },
      "language": "en",
      "prompt": "A deep-sea monster rises from the waters once every hundred years to feast on a ship and sate its hunger. Over three hundred years, it has consumed 847 people. Ships have been built larger over time, so each new ship has twice as many people as the last ship. How many people were on the ship the monster ate in the first hundred years?",
      "context": [],
      "environment_note": "Self-contained arithmetic; no external environment needed.",
      "rubric": {
        "markdown": "# Recover the first ship's passenger count\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct requested quantity\n\n**ID:** answer\n\n**Criterion:** Gives the requested quantity as 121, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** The final answer is mathematically equivalent to the stated quantity.\n- **0:** The answer is incorrect, absent, or contradicted by another final answer.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_answer\", \"arithmetic\\_check\"\\]\n\n### Valid reasoning when supplied\n\n**ID:** faithfulness\n\n**Criterion:** Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Explanation, if present, is valid; a correct answer without a derivation also passes.\n- **0:** The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"reviewed\\_alternative\\_solutions\"\\]\n",
        "raw": {
          "title": "Recover the first ship's passenger count",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "answer",
              "criterion": "Gives the requested quantity as 121, with no conflicting final answer. Equivalent numeric notation and units clear from context are acceptable.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "The final answer is mathematically equivalent to the stated quantity.",
                "0": "The answer is incorrect, absent, or contradicted by another final answer."
              },
              "evidence_refs": [
                "source_answer",
                "arithmetic_check"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            },
            {
              "id": "faithfulness",
              "criterion": "Any explanation supplied uses the quantities and relationships in the question consistently. The question does not request a derivation, so a correct concise answer receives full credit; do not require intermediate steps or a named method.",
              "weight": 2.0,
              "grading_anchors": {
                "1": "Explanation, if present, is valid; a correct answer without a derivation also passes.",
                "0": "The explanation asserts an incorrect relationship or invalid calculation; an incorrect answer-only response also fails."
              },
              "evidence_refs": [
                "task",
                "reviewed_alternative_solutions"
              ],
              "status": "provisional",
              "method": "llm",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "answer": {
            "title": "Correct requested quantity",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "faithfulness": {
            "title": "Valid reasoning when supplied",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "121",
        "solution": "Let S be the number of people on the first hundred years’ ship.\nThe second hundred years’ ship had twice as many as the first, so it had 2S people.\nThe third hundred years’ ship had twice as many as the second, so it had 2 * 2S = <<2*2=4>>4S people.\nAll the ships had S + 2S + 4S = 7S = 847 people.\nThus, the ship that the monster ate in the first hundred years had S = 847 / 7 = <<847/7=121>>121 people on it.\n#### 121",
        "tests": [
          "847 / (1 + 2 + 4) == 121",
          "847 / 7 == 121",
          "121 == 121"
        ]
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "A reference answer independently cross-checked by arithmetic.",
          "Accepts concise answers and different valid derivations."
        ],
        "risks": [
          "Matching a final number cannot certify arbitrary mathematical reasoning.",
          "These are deliberately simple conversion demonstrations, not evidence of advanced-math coverage."
        ],
        "must_hold": [
          "The requested quantity is 121.",
          "Any volunteered reasoning must be sound."
        ],
        "may_vary": [
          "Algebra, arithmetic, proportions, or a concise answer.",
          "Wording, notation, layout, and order of valid steps."
        ],
        "proposed_changes": [
          "Do not turn every reference-solution step into a mandatory criterion.",
          "Verify explanations independently when extending this to proofs."
        ]
      },
      "probes": [
        {
          "label": "Solve a linear equation",
          "kind": "valid_alternative",
          "response": "Let the first ship carry x people. The next two carry 2x and 4x. Thus 7x = 847 and x = 121 people.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 121. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 121.0,
            "reference": 121,
            "passed": true
          }
        },
        {
          "label": "Work backward through the ratios",
          "kind": "valid_alternative",
          "response": "The three passenger counts are in the ratio 1:2:4. Splitting 847 into seven equal shares gives 121 per share. The first ship had 121 people; the next two had 242 and 484.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 121. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 121.0,
            "reference": 121,
            "passed": true
          }
        },
        {
          "label": "Concise answer",
          "kind": "valid_alternative",
          "response": "121 people.",
          "is_full_response": true,
          "expected": "Accept; full credit without matching a particular derivation.",
          "verification": "executed",
          "why": "Exact numeric check gives 121. The link from the displayed prose to this calculation is editorially reviewed.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 121,
            "reference": 121,
            "passed": true
          }
        },
        {
          "label": "Incorrect reasoning control",
          "kind": "invalid_control",
          "response": "The three ships had equal-sized groups, so the first carried 847/3 people.",
          "is_full_response": true,
          "expected": "Reject: incorrect quantity and reasoning.",
          "verification": "executed",
          "why": "The displayed calculation produces a number unequal to the verified source answer.",
          "result": {
            "check": "exact_numeric_answer",
            "observed": 282.3333333333333,
            "reference": 121,
            "passed": false
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Exact arithmetic checks plus editorial review of displayed responses; no model rubric-judge run.",
        "checks": [
          {
            "name": "Solve a linear equation",
            "passed": true,
            "detail": "Authored arithmetic expression 847 / (1 + 2 + 4) evaluates to 121; prose reviewed separately."
          },
          {
            "name": "Work backward through the ratios",
            "passed": true,
            "detail": "Authored arithmetic expression 847 / 7 evaluates to 121; prose reviewed separately."
          },
          {
            "name": "Concise answer",
            "passed": true,
            "detail": "Authored arithmetic expression 121 evaluates to 121; prose reviewed separately."
          },
          {
            "name": "Wrong-answer control is rejected",
            "passed": true,
            "detail": "282.3333333333333 differs from the reference answer 121."
          }
        ],
        "limitations": [
          "Only the numeric check is executable. Natural-language reasoning and the draft rubric were reviewed editorially.",
          "The three correct responses are constructed coverage probes, not sampled student completions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:mimo:s3k_1927_last_mile_logistics_zh_t2_rl_003",
      "title": "车辆整改：依赖关系与复运条件",
      "source": {
        "dataset": "XiaomiMiMo/MiMo-V2.6-RL-oss",
        "source_name": "mimo",
        "url": "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss",
        "revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "original_inventory_revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Original inventory verified Hugging Face commit",
        "split": "general/train",
        "row_id": "s3k_1927_last_mile_logistics_zh_t2_rl_003",
        "license": "Apache-2.0",
        "pool": "current",
        "source_content_sha256": "ade5348ace27b5b7891b2b70beaf8174a6d133d1bc82b7b4b1d355d7a530e616",
        "source_record_fingerprint": "c3e29c49703747e8a1b6bfcbe8ed854a3da7e942aa32d4b49b156f8812bba442",
        "source_row_index": 901,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": false,
        "synthetic_claim_from_import": true,
        "attribution": "Source task and rubric: mimo; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "planning_scheduling",
        "leaf": "planning_scheduling.dependency_workflow_planning",
        "confidence": "editorial_provisional",
        "reason": "The requested plan orders review checkpoints and responsibilities with prerequisites and escalation conditions."
      },
      "language": "zh",
      "prompt": "user: 请根据 `workspace/03_合规报告/京东物流南安车队车辆整改闭环会议纪要.docx`，并结合 `workspace/03_合规报告/京东物流南安车队2025年春运车辆合规处置报告.docx` 中的责任边界和闭环口径，为车队负责人整理一份 2025 年 1 月 20 日整改协调会之后的执行安排。以会议纪要确定的会后复核日程为准，按时间先后写清每个复核节点的日期时间、核验对象、牵头或直接负责的角色，以及发现未按期完成事项时应采取的冻结、转停运或升级动作；同时把维修协调、车辆技术复检、安全配置核验、调度冻结和合规升级之间的责任衔接说明白。\n\n最终回复还要明确提醒负责人：维修完成、现场复检合格、线上年审反馈分别代表什么，为什么三者不能互相替代；涉及网上年审时，应说明材料提交与受理时点的前提及监管办理时限。只需提交分析结论，不要修改 workspace 文件。",
      "context": [],
      "environment_note": "Original remote workspace and tools are required and are not included or executed. Response probes compare representations against source anchors only; no environment success is claimed.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### 最终回复是否正确给出第一次专项复核的日期时间、核验对象和核心责任角色？\n\n**ID:** first-review-node\n\n**Criterion:** 最终回复是否正确给出第一次专项复核的日期时间、核验对象和核心责任角色？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：第一次专项复核为 2025年1月22日10:00，重点核对12辆停运车逐车的维修工单、零部件更换凭证和维修/复检证据链或复检状态。专项复核与升级由合规专员林志宏牵头；维修材料由黄伟杰及外委维修单位负责，车辆技术复检由黄思远负责，涉及随车安全配置时由许德诚复核。答案须给出准确时点和核验范围，并至少正确识别林志宏的牵头职责以及黄思远或黄伟杰之一的直接职责；角色称谓或姓名的等价表达可接受。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否正确给出第二次复核的日期时间、核验对象和核心责任角色？\n\n**ID:** second-review-node\n\n**Criterion:** 最终回复是否正确给出第二次复核的日期时间、核验对象和核心责任角色？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：第二次复核为 2025年1月24日16:00，核验线上年审材料提交完整性、提交回执/受理证据及调度冻结清单。道路运输证和网上年审由合规专员林志宏主责，调度冻结清单由调度组长蔡婉宁维护，黄思远配合证照/技术材料核对。答案须给出准确时点与两类核验对象，并正确对应林志宏和蔡婉宁的职责。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否正确给出区域抽检的日期时间、抽检范围和责任衔接？\n\n**ID:** regional-audit-node\n\n**Criterion:** 最终回复是否正确给出区域抽检的日期时间、抽检范围和责任衔接？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：区域抽检为 2025年1月27日09:30，由区域合规管理人员抽检一车一档目录、纸质原件与电子扫描件的一致性。黄思远负责一车一档归集/档案核验，许德诚负责抽查，林志宏负责合规升级联络。答案须准确给出时点、抽检对象，并识别区域合规管理人员及至少一名内部责任角色的衔接。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否正确区分未按期完成时对立即停运车辆和限期整改车辆的处置与升级动作？\n\n**ID:** overdue-escalation\n\n**Criterion:** 最终回复是否正确区分未按期完成时对立即停运车辆和限期整改车辆的处置与升级动作？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：未按期完成的立即停运车辆继续保持调度冻结/不得派单，并向泉州区域运营保障部升级报告；限期整改车辆若未完成关键安全项，应转入停运清单（会议另规定限期整改逾期自动转为停运）。例外事项由车队负责人陈雅雯组织书面确认。答案须至少正确写出前两类差异化动作，不能笼统写成仅提醒整改。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否正确说明维修、技术复检、安全配置、调度冻结和合规升级的责任链？\n\n**ID:** responsibility-chain\n\n**Criterion:** 最终回复是否正确说明维修、技术复检、安全配置、调度冻结和合规升级的责任链？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：黄伟杰负责维修协调、工单及竣工合格证收集；黄思远负责车辆技术复检及档案核验；许德诚负责安全配置核验与抽查；蔡婉宁负责调度冻结、钥匙证照交接并每日17:30前更新限制状态；林志宏负责合规口径复核、专项复核与升级联络。五类职责应基本对应，允许用岗位替代姓名，但不能把维修完成者写成最终解封批准人。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否明确区分维修完成、现场复检合格和线上年审反馈，且没有把任一节点当成完整闭环或自动复运？\n\n**ID:** independent-milestones\n\n**Criterion:** 最终回复是否明确区分维修完成、现场复检合格和线上年审反馈，且没有把任一节点当成完整闭环或自动复运？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：维修完成只证明维修执行及工单/竣工资料形成；现场复检合格证明缺陷经技术或安全核验消除；线上年审反馈证明证照审验流程的受理或审验结果。三者须分别登记、不能互相替代。完整闭环还要求材料可追溯归档并解除相应调度限制，立即停运车辆须满足证照或审验结论、维修竣工证明、现场复检和归档等条件后才可申请解封。答案须表达三类节点独立且单一节点不等于闭环/复运。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否正确说明线上年审材料提交、受理时点和监管办理时限？\n\n**ID:** online-review-timing\n\n**Criterion:** 最终回复是否正确说明线上年审材料提交、受理时点和监管办理时限？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实文档锚点：线上年审材料应在复检合格后当日上传；只有完整材料才计入受理完成时点，材料不完整不计入；南安市交通运输事业发展中心收到完整材料后3个工作日内完成审验。答案须包含这三个口径，日期格式和自然语言表述可不同。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "最终回复是否正确给出第一次专项复核的日期时间、核验对象和核心责任角色？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：第一次专项复核为 2025年1月22日10:00，重点核对12辆停运车逐车的维修工单、零部件更换凭证和维修/复检证据链或复检状态。专项复核与升级由合规专员林志宏牵头；维修材料由黄伟杰及外委维修单位负责，车辆技术复检由黄思远负责，涉及随车安全配置时由许德诚复核。答案须给出准确时点和核验范围，并至少正确识别林志宏的牵头职责以及黄思远或黄伟杰之一的直接职责；角色称谓或姓名的等价表达可接受。"
              },
              "id": "first-review-node",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否正确给出第二次复核的日期时间、核验对象和核心责任角色？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：第二次复核为 2025年1月24日16:00，核验线上年审材料提交完整性、提交回执/受理证据及调度冻结清单。道路运输证和网上年审由合规专员林志宏主责，调度冻结清单由调度组长蔡婉宁维护，黄思远配合证照/技术材料核对。答案须给出准确时点与两类核验对象，并正确对应林志宏和蔡婉宁的职责。"
              },
              "id": "second-review-node",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否正确给出区域抽检的日期时间、抽检范围和责任衔接？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：区域抽检为 2025年1月27日09:30，由区域合规管理人员抽检一车一档目录、纸质原件与电子扫描件的一致性。黄思远负责一车一档归集/档案核验，许德诚负责抽查，林志宏负责合规升级联络。答案须准确给出时点、抽检对象，并识别区域合规管理人员及至少一名内部责任角色的衔接。"
              },
              "id": "regional-audit-node",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否正确区分未按期完成时对立即停运车辆和限期整改车辆的处置与升级动作？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：未按期完成的立即停运车辆继续保持调度冻结/不得派单，并向泉州区域运营保障部升级报告；限期整改车辆若未完成关键安全项，应转入停运清单（会议另规定限期整改逾期自动转为停运）。例外事项由车队负责人陈雅雯组织书面确认。答案须至少正确写出前两类差异化动作，不能笼统写成仅提醒整改。"
              },
              "id": "overdue-escalation",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否正确说明维修、技术复检、安全配置、调度冻结和合规升级的责任链？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：黄伟杰负责维修协调、工单及竣工合格证收集；黄思远负责车辆技术复检及档案核验；许德诚负责安全配置核验与抽查；蔡婉宁负责调度冻结、钥匙证照交接并每日17:30前更新限制状态；林志宏负责合规口径复核、专项复核与升级联络。五类职责应基本对应，允许用岗位替代姓名，但不能把维修完成者写成最终解封批准人。"
              },
              "id": "responsibility-chain",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否明确区分维修完成、现场复检合格和线上年审反馈，且没有把任一节点当成完整闭环或自动复运？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：维修完成只证明维修执行及工单/竣工资料形成；现场复检合格证明缺陷经技术或安全核验消除；线上年审反馈证明证照审验流程的受理或审验结果。三者须分别登记、不能互相替代。完整闭环还要求材料可追溯归档并解除相应调度限制，立即停运车辆须满足证照或审验结论、维修竣工证明、现场复检和归档等条件后才可申请解封。答案须表达三类节点独立且单一节点不等于闭环/复运。"
              },
              "id": "independent-milestones",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否正确说明线上年审材料提交、受理时点和监管办理时限？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实文档锚点：线上年审材料应在复检合格后当日上传；只有完整材料才计入受理完成时点，材料不完整不计入；南安市交通运输事业发展中心收到完整材料后3个工作日内完成审验。答案须包含这三个口径，日期格式和自然语言表述可不同。"
              },
              "id": "online-review-timing",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 7,
        "source_original_items": [
          {
            "files": [
              "answer.md"
            ],
            "id": "first-review-node",
            "method": "llm",
            "pass_anchor": "真实文档锚点：第一次专项复核为 2025年1月22日10:00，重点核对12辆停运车逐车的维修工单、零部件更换凭证和维修/复检证据链或复检状态。专项复核与升级由合规专员林志宏牵头；维修材料由黄伟杰及外委维修单位负责，车辆技术复检由黄思远负责，涉及随车安全配置时由许德诚复核。答案须给出准确时点和核验范围，并至少正确识别林志宏的牵头职责以及黄思远或黄伟杰之一的直接职责；角色称谓或姓名的等价表达可接受。",
            "question": "最终回复是否正确给出第一次专项复核的日期时间、核验对象和核心责任角色？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "second-review-node",
            "method": "llm",
            "pass_anchor": "真实文档锚点：第二次复核为 2025年1月24日16:00，核验线上年审材料提交完整性、提交回执/受理证据及调度冻结清单。道路运输证和网上年审由合规专员林志宏主责，调度冻结清单由调度组长蔡婉宁维护，黄思远配合证照/技术材料核对。答案须给出准确时点与两类核验对象，并正确对应林志宏和蔡婉宁的职责。",
            "question": "最终回复是否正确给出第二次复核的日期时间、核验对象和核心责任角色？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "regional-audit-node",
            "method": "llm",
            "pass_anchor": "真实文档锚点：区域抽检为 2025年1月27日09:30，由区域合规管理人员抽检一车一档目录、纸质原件与电子扫描件的一致性。黄思远负责一车一档归集/档案核验，许德诚负责抽查，林志宏负责合规升级联络。答案须准确给出时点、抽检对象，并识别区域合规管理人员及至少一名内部责任角色的衔接。",
            "question": "最终回复是否正确给出区域抽检的日期时间、抽检范围和责任衔接？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "overdue-escalation",
            "method": "llm",
            "pass_anchor": "真实文档锚点：未按期完成的立即停运车辆继续保持调度冻结/不得派单，并向泉州区域运营保障部升级报告；限期整改车辆若未完成关键安全项，应转入停运清单（会议另规定限期整改逾期自动转为停运）。例外事项由车队负责人陈雅雯组织书面确认。答案须至少正确写出前两类差异化动作，不能笼统写成仅提醒整改。",
            "question": "最终回复是否正确区分未按期完成时对立即停运车辆和限期整改车辆的处置与升级动作？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "responsibility-chain",
            "method": "llm",
            "pass_anchor": "真实文档锚点：黄伟杰负责维修协调、工单及竣工合格证收集；黄思远负责车辆技术复检及档案核验；许德诚负责安全配置核验与抽查；蔡婉宁负责调度冻结、钥匙证照交接并每日17:30前更新限制状态；林志宏负责合规口径复核、专项复核与升级联络。五类职责应基本对应，允许用岗位替代姓名，但不能把维修完成者写成最终解封批准人。",
            "question": "最终回复是否正确说明维修、技术复检、安全配置、调度冻结和合规升级的责任链？",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "independent-milestones",
            "method": "llm",
            "pass_anchor": "真实文档锚点：维修完成只证明维修执行及工单/竣工资料形成；现场复检合格证明缺陷经技术或安全核验消除；线上年审反馈证明证照审验流程的受理或审验结果。三者须分别登记、不能互相替代。完整闭环还要求材料可追溯归档并解除相应调度限制，立即停运车辆须满足证照或审验结论、维修竣工证明、现场复检和归档等条件后才可申请解封。答案须表达三类节点独立且单一节点不等于闭环/复运。",
            "question": "最终回复是否明确区分维修完成、现场复检合格和线上年审反馈，且没有把任一节点当成完整闭环或自动复运？",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "online-review-timing",
            "method": "llm",
            "pass_anchor": "真实文档锚点：线上年审材料应在复检合格后当日上传；只有完整材料才计入受理完成时点，材料不完整不计入；南安市交通运输事业发展中心收到完整材料后3个工作日内完成审验。答案须包含这三个口径，日期格式和自然语言表述可不同。",
            "question": "最终回复是否正确说明线上年审材料提交、受理时点和监管办理时限？",
            "tier": "important"
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Weighted sum divided by all criterion weights; missing source weights are 1. Tiers are metadata, not implicit gates.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Accepts roles instead of names and flexible date phrasing.",
          "Separates repair, inspection, annual review, and complete release conditions."
        ],
        "risks": [
          "Named meetings and source documents are absent; deadlines and roles remain unverified.",
          "A chronological summary without dependencies could superficially resemble a workable plan."
        ],
        "must_hold": [
          "Preserve source-controlled checkpoints, responsibility boundaries and overdue escalation.",
          "Do not equate repair completion with authorization to resume service."
        ],
        "may_vary": [
          "Timeline, responsibility matrix, or concise prose, as long as time order is clear.",
          "Role names versus personal names when identity and accountability remain unambiguous."
        ],
        "proposed_changes": [
          "Separate hard dates and prerequisites from presentation; test a timeline and a role-centered plan.",
          "Require the actual controlled documents before certifying factual completeness."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: chronological checklist",
          "kind": "valid_alternative",
          "response": "按 1月22日10:00、1月24日16:00、1月27日09:30 的来源节点组织清单，逐项列核验材料、责任人及逾期处置；单独列明维修、复检、年审反馈和解封的依赖。",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Matches source-anchor ordering, pending verification of the documents."
        },
        {
          "label": "Sketch B: responsibility matrix plus dates",
          "kind": "valid_alternative",
          "response": "以维修协调、技术复检、安全配置、调度冻结、合规升级为列，以三个复核时间点为行；每个单元明确交接材料与阻塞条件，并提醒完整受理后才开始监管办理时限。",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Different representation can preserve the same executable dependencies."
        },
        {
          "label": "Invalid: automatic release",
          "kind": "invalid_control",
          "response": "维修单位说修好了，就立即解除所有调度冻结；线上年审和复检可以之后补做。",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Ignores independent release prerequisites."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605427",
      "title": "Washington July 4: feasible itineraries, different choices",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605427",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "4f97d25fbe5c182eadcd41658b2733079cad1630dde91de2af061845fc01f33d",
        "source_row_index": 0,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "analysis_coordination",
        "family": "planning_scheduling",
        "leaf": "planning_scheduling.schedule_resource_allocation",
        "confidence": "editorial_provisional",
        "reason": "A dated itinerary must allocate time across activities, travel, meals, and contingencies."
      },
      "language": "en",
      "prompt": "I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\n\n**ID:** c1\n\n**Criterion:** The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains clear section headers for parts of the schedule (e.g., Morning, Afternoon, Fireworks, Lunch by the White House).\n\n**ID:** c2\n\n**Criterion:** The response contains clear section headers for parts of the schedule (e.g., Morning, Afternoon, Fireworks, Lunch by the White House).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes officially recurring DC events (e.g., National Independence Day Parade, A Capitol Fourth Concert, National Mall Fireworks, Mount Vernon tour)\n\n**ID:** c3\n\n**Criterion:** The response includes officially recurring DC events (e.g., National Independence Day Parade, A Capitol Fourth Concert, National Mall Fireworks, Mount Vernon tour)\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response suggests a start time and an end time for every activity listed.  \n\n**ID:** c4\n\n**Criterion:** The response suggests a start time and an end time for every activity listed.  \n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response proposes a minimum of four distinct activities or stops (e.g., Independence Day Parade, A Capital Fourth, arriving for fireworks on the National Mall by 8 PM, a tour of Mount Vernon). \n\n**ID:** c5\n\n**Criterion:** The response proposes a minimum of four distinct activities or stops (e.g., Independence Day Parade, A Capital Fourth, arriving for fireworks on the National Mall by 8 PM, a tour of Mount Vernon). \n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response comments on the proximity of this year's 4th of July to the weekend (5–6 July 2025) and suggests additional DC activities (e.g., Dumbarton Oaks, Smithsonians, Great Falls hike, walking about the Georgetown Neighborhood).  \n\n**ID:** c6\n\n**Criterion:** The response comments on the proximity of this year's 4th of July to the weekend (5–6 July 2025) and suggests additional DC activities (e.g., Dumbarton Oaks, Smithsonians, Great Falls hike, walking about the Georgetown Neighborhood).  \n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The reponse contains at least one sentence explaining why the visitor might enjoy it (Mount Vernon is George Washington's home and leans into the holiday theme, the fireworks costs more than $1 million, the Declaration of Independence is stored at the National Archives so the dramatic reading is historical, going to the The Tombs is a Georgetown tradition and the food is affordable).\n\n**ID:** c7\n\n**Criterion:** The reponse contains at least one sentence explaining why the visitor might enjoy it (Mount Vernon is George Washington's home and leans into the holiday theme, the fireworks costs more than $1 million, the Declaration of Independence is stored at the National Archives so the dramatic reading is historical, going to the The Tombs is a Georgetown tradition and the food is affordable).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The reponse recommends at least one activity that is free of charge to attend (e.g., watching the fireworks from a bridge, \"A Capitol Fourth\", National Independence Day Parade, using the free metro service that evening.)\n\n**ID:** c8\n\n**Criterion:** The reponse recommends at least one activity that is free of charge to attend (e.g., watching the fireworks from a bridge, \"A Capitol Fourth\", National Independence Day Parade, using the free metro service that evening.)\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides at least one indoor backup option in case of bad weather (e.g., Natural History Musuem, African American History Museum, Steven F. Udvar-Hazy Center, Founding Farmers or another DC specific restaurant.)  \n\n**ID:** c9\n\n**Criterion:** The response provides at least one indoor backup option in case of bad weather (e.g., Natural History Musuem, African American History Museum, Steven F. Udvar-Hazy Center, Founding Farmers or another DC specific restaurant.)  \n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes lunch and dinner dining suggestion located inside Washington, DC or Alexandria/Arlingon or Silver Spring (e.g., Founding Farmers, Le Diplomate, Ben's Chili Bowl, Busboys and Poets).\n\n**ID:** c10\n\n**Criterion:** The response includes lunch and dinner dining suggestion located inside Washington, DC or Alexandria/Arlingon or Silver Spring (e.g., Founding Farmers, Le Diplomate, Ben's Chili Bowl, Busboys and Poets).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explicitly states that exact 2025 event times are tentative and should be confirmed closer to the date.\n\n**ID:** c11\n\n**Criterion:** The response explicitly states that exact 2025 event times are tentative and should be confirmed closer to the date.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response warns about large crowds or security screenings on the National Mall on July 4.\n\n**ID:** c12\n\n**Criterion:** The response warns about large crowds or security screenings on the National Mall on July 4.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response recommends using personal fireworks use or any other illegal activity.\n\n**ID:** c13\n\n**Criterion:** The response recommends using personal fireworks use or any other illegal activity.\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides estimated travel time and transit advice between activites (e.g, 30 minutes to take the green line, Uber prices are historically $20 from Alexandria to the Smitsonian mall, Mount Vernon is far from the Mall and should require at least an hour of travel time, SmarTrip cards can be purhased at pharmacies). \n\n**ID:** c14\n\n**Criterion:** The response provides estimated travel time and transit advice between activites (e.g, 30 minutes to take the green line, Uber prices are historically $20 from Alexandria to the Smitsonian mall, Mount Vernon is far from the Mall and should require at least an hour of travel time, SmarTrip cards can be purhased at pharmacies). \n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes at least one activity suitable for families with children (e.g., watching the fireworks from a bridge away from the densest crowd, National Independence Day parade, Independence Day Junior Ranger, walking around Georgetown or the Wharf).\n\n**ID:** c15\n\n**Criterion:** The response includes at least one activity suitable for families with children (e.g., watching the fireworks from a bridge away from the densest crowd, National Independence Day parade, Independence Day Junior Ranger, walking around Georgetown or the Wharf).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions when the fireworks start (9 PM).\n\n**ID:** c16\n\n**Criterion:** The response mentions when the fireworks start (9 PM).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses different locations to watch the fireworks (e.g., National Mall, National Cathedral, Key Bridge, Long Bridge Park).\n\n**ID:** c17\n\n**Criterion:** The response discusses different locations to watch the fireworks (e.g., National Mall, National Cathedral, Key Bridge, Long Bridge Park).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains external promotional links or affiliate marketing codes.\n\n**ID:** c18\n\n**Criterion:** The response contains external promotional links or affiliate marketing codes.\n\n**Points:** -3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response acknowledges possible schedule changes due to weather or security and advises checking official sources.\n\n**ID:** c19\n\n**Criterion:** The response acknowledges possible schedule changes due to weather or security and advises checking official sources.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions a city outside of Washington DC, Maryland, or Virginia (e.g., Philadelphia, New York, or Baltimore, McLean or other distant towns are acceptable as a family friendly alternative).\n\n**ID:** c20\n\n**Criterion:** The response mentions a city outside of Washington DC, Maryland, or Virginia (e.g., Philadelphia, New York, or Baltimore, McLean or other distant towns are acceptable as a family friendly alternative).\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response recommends events from years other than 2025 (e.g., seeing a miltiary parade, information about \"A Capital Fourth\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\n\n**ID:** c21\n\n**Criterion:** The response recommends events from years other than 2025 (e.g., seeing a miltiary parade, information about \"A Capital Fourth\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\n\n**Points:** -4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Planning and scheduling\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response contains clear section headers for parts of the schedule (e.g., Morning, Afternoon, Fireworks, Lunch by the White House).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response includes officially recurring DC events (e.g., National Independence Day Parade, A Capitol Fourth Concert, National Mall Fireworks, Mount Vernon tour)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response suggests a start time and an end time for every activity listed.  ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response proposes a minimum of four distinct activities or stops (e.g., Independence Day Parade, A Capital Fourth, arriving for fireworks on the National Mall by 8 PM, a tour of Mount Vernon). ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response comments on the proximity of this year's 4th of July to the weekend (5–6 July 2025) and suggests additional DC activities (e.g., Dumbarton Oaks, Smithsonians, Great Falls hike, walking about the Georgetown Neighborhood).  ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The reponse contains at least one sentence explaining why the visitor might enjoy it (Mount Vernon is George Washington's home and leans into the holiday theme, the fireworks costs more than $1 million, the Declaration of Independence is stored at the National Archives so the dramatic reading is historical, going to the The Tombs is a Georgetown tradition and the food is affordable).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The reponse recommends at least one activity that is free of charge to attend (e.g., watching the fireworks from a bridge, \"A Capitol Fourth\", National Independence Day Parade, using the free metro service that evening.)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response provides at least one indoor backup option in case of bad weather (e.g., Natural History Musuem, African American History Museum, Steven F. Udvar-Hazy Center, Founding Farmers or another DC specific restaurant.)  ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes lunch and dinner dining suggestion located inside Washington, DC or Alexandria/Arlingon or Silver Spring (e.g., Founding Farmers, Le Diplomate, Ben's Chili Bowl, Busboys and Poets).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response explicitly states that exact 2025 event times are tentative and should be confirmed closer to the date.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response warns about large crowds or security screenings on the National Mall on July 4.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response recommends using personal fireworks use or any other illegal activity.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "The response provides estimated travel time and transit advice between activites (e.g, 30 minutes to take the green line, Uber prices are historically $20 from Alexandria to the Smitsonian mall, Mount Vernon is far from the Mall and should require at least an hour of travel time, SmarTrip cards can be purhased at pharmacies). ",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response includes at least one activity suitable for families with children (e.g., watching the fireworks from a bridge away from the densest crowd, National Independence Day parade, Independence Day Junior Ranger, walking around Georgetown or the Wharf).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response mentions when the fireworks start (9 PM).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response discusses different locations to watch the fireworks (e.g., National Mall, National Cathedral, Key Bridge, Long Bridge Park).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response contains external promotional links or affiliate marketing codes.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": -3.0
            },
            {
              "criterion": "The response acknowledges possible schedule changes due to weather or security and advises checking official sources.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response mentions a city outside of Washington DC, Maryland, or Virginia (e.g., Philadelphia, New York, or Baltimore, McLean or other distant towns are acceptable as a family friendly alternative).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "The response recommends events from years other than 2025 (e.g., seeing a miltiary parade, information about \"A Capital Fourth\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": -4.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 21,
        "source_original_items": [
          {
            "axis": "Explicit Criteria",
            "criterion": "The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025",
            "weight": 5.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response contains clear section headers for parts of the schedule (e.g., Morning, Afternoon, Fireworks, Lunch by the White House).",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response includes officially recurring DC events (e.g., National Independence Day Parade, A Capitol Fourth Concert, National Mall Fireworks, Mount Vernon tour)",
            "weight": 5.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response suggests a start time and an end time for every activity listed.  ",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response proposes a minimum of four distinct activities or stops (e.g., Independence Day Parade, A Capital Fourth, arriving for fireworks on the National Mall by 8 PM, a tour of Mount Vernon). ",
            "weight": 1.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response comments on the proximity of this year's 4th of July to the weekend (5–6 July 2025) and suggests additional DC activities (e.g., Dumbarton Oaks, Smithsonians, Great Falls hike, walking about the Georgetown Neighborhood).  ",
            "weight": 4.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The reponse contains at least one sentence explaining why the visitor might enjoy it (Mount Vernon is George Washington's home and leans into the holiday theme, the fireworks costs more than $1 million, the Declaration of Independence is stored at the National Archives so the dramatic reading is historical, going to the The Tombs is a Georgetown tradition and the food is affordable).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The reponse recommends at least one activity that is free of charge to attend (e.g., watching the fireworks from a bridge, \"A Capitol Fourth\", National Independence Day Parade, using the free metro service that evening.)",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response provides at least one indoor backup option in case of bad weather (e.g., Natural History Musuem, African American History Museum, Steven F. Udvar-Hazy Center, Founding Farmers or another DC specific restaurant.)  ",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes lunch and dinner dining suggestion located inside Washington, DC or Alexandria/Arlingon or Silver Spring (e.g., Founding Farmers, Le Diplomate, Ben's Chili Bowl, Busboys and Poets).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explicitly states that exact 2025 event times are tentative and should be confirmed closer to the date.",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response warns about large crowds or security screenings on the National Mall on July 4.",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response recommends using personal fireworks use or any other illegal activity.",
            "weight": -5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response provides estimated travel time and transit advice between activites (e.g, 30 minutes to take the green line, Uber prices are historically $20 from Alexandria to the Smitsonian mall, Mount Vernon is far from the Mall and should require at least an hour of travel time, SmarTrip cards can be purhased at pharmacies). ",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes at least one activity suitable for families with children (e.g., watching the fireworks from a bridge away from the densest crowd, National Independence Day parade, Independence Day Junior Ranger, walking around Georgetown or the Wharf).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response mentions when the fireworks start (9 PM).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses different locations to watch the fireworks (e.g., National Mall, National Cathedral, Key Bridge, Long Bridge Park).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response contains external promotional links or affiliate marketing codes.",
            "weight": -3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response acknowledges possible schedule changes due to weather or security and advises checking official sources.",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response mentions a city outside of Washington DC, Maryland, or Virginia (e.g., Philadelphia, New York, or Baltimore, McLean or other distant towns are acceptable as a family friendly alternative).",
            "weight": -5.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response recommends events from years other than 2025 (e.g., seeing a miltiary parade, information about \"A Capital Fourth\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).",
            "weight": -4.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Checks travel time, crowd/security concerns, backup options, and schedule changes.",
          "Allows alternatives for meals, activities, and viewing locations."
        ],
        "risks": [
          "c16 hardcodes a 9 PM fireworks start rather than requiring the verified 2025 schedule.",
          "Earliest/latest coverage and universal start/end times may encourage false precision.",
          "Historical 2025 hours, access and transit have not been verified for this gallery."
        ],
        "must_hold": [
          "Stay grounded in the requested date and region, with feasible transfers and clear timing uncertainty.",
          "Explain activity choices, crowd considerations and contingency plans."
        ],
        "may_vary": [
          "Museum-focused versus outdoor route, meal choices, viewing location and optional weekend extension.",
          "Conservative time windows instead of invented exact times where official schedules are uncertain."
        ],
        "proposed_changes": [
          "Replace hardcoded fireworks time with agreement with a pinned official 2025 source.",
          "Evaluate schedule feasibility rather than matching a fixed canonical attraction sequence.",
          "Distinguish attendance instructions from merely mentioning out-of-region places."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: compact central route",
          "kind": "valid_alternative",
          "response": "Choose a central-DC museum and parade itinerary, reserve time for meals and security, then use one verified fireworks viewing site with a weather alternative nearby. Check the official 2025 schedules before assigning times.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Keeps travel short; precise dated claims require external verification."
        },
        {
          "label": "Sketch B: low-crowd route with weekend extension",
          "kind": "valid_alternative",
          "response": "Prioritize a less congested viewing location and fewer July 4 stops; move distant historic attractions to July 5 or 6. Include transit and security buffers, accessible alternatives and a rain plan, all against archived official schedules.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Different valid priorities need not match a single attraction list."
        },
        {
          "label": "Invalid: impossible transfer",
          "kind": "invalid_control",
          "response": "Stay at Mount Vernon until the moment the National Mall fireworks begin, then walk to the Mall in five minutes.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Cannot satisfy geographical and timing feasibility."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605355",
      "title": "AI drug discovery: evidence across the pipeline",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605355",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "b98475fa20f82f49702f0b26daff170e90762f4f825bc6ba697595a352025269",
        "source_row_index": 2,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "evidence_investigation",
        "family": "research_document_synthesis",
        "leaf": "research_document_synthesis.grounded_explanation_synthesis",
        "confidence": "editorial_provisional",
        "reason": "The deliverable synthesizes technical and real-world evidence across drug-development stages for a specified audience."
      },
      "language": "en",
      "prompt": "Write a synthesis report on the applications of AI in drug discovery for a technical audience unfamiliar with biology. It should cover the main applications of AI in every stage of the drug discovery process, the latest technological advancements, challenges, and current adoption in the real world.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response, for each stage, names at least one specific AI model class or method other than generative models (e.g., VAEs, Foundation Models) (e.g., (1) CNNs (Convolutional Neural Networks), (2) GNNs, (3) Classical ML methods like random forests, (4) knowledge graphs, (5) Hybrid AI-physics Models).\n\n**ID:** c1\n\n**Criterion:** The response, for each stage, names at least one specific AI model class or method other than generative models (e.g., VAEs, Foundation Models) (e.g., (1) CNNs (Convolutional Neural Networks), (2) GNNs, (3) Classical ML methods like random forests, (4) knowledge graphs, (5) Hybrid AI-physics Models).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes at least one specific AI application for EACH of the following drug-discovery stages:\n\n**ID:** c2\n\n**Criterion:** The response describes at least one specific AI application for EACH of the following drug-discovery stages:<br><br><br>(1) Target Identification<br><br><br>(2) Hit/Lead Discovery or Virtual Screening<br><br><br>(3) Lead Optimization<br><br><br>(4) Preclinical Safety / ADMET Prediction<br><br><br>(5) Clinical Trial Design / Patient Stratification<br><br><br>(6) Post-market Surveillance / Pharmacovigilance\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides brief (≤20 words) definitions for all specialized terms, acronyms, and each of the six drug-discovery stages before discussing their AI applications.\n\n**ID:** c3\n\n**Criterion:** The response provides brief (≤20 words) definitions for all specialized terms, acronyms, and each of the six drug-discovery stages before discussing their AI applications.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes the target audience (technical, non-biology) explicitly or implicitly in the introduction or tone.\n\n**ID:** c4\n\n**Criterion:** The response describes the target audience (technical, non-biology) explicitly or implicitly in the introduction or tone.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses deterministic language (“will”, “always”, “guarantee”, “ensure”) for speculative claims about future directions or unverified findings.\n\n**ID:** c5\n\n**Criterion:** The response uses deterministic language (“will”, “always”, “guarantee”, “ensure”) for speculative claims about future directions or unverified findings.\n\n**Points:** -4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains internal contradictions (e.g., calls a Phase II drug “approved” elsewhere).\n\n**ID:** c6\n\n**Criterion:** The response contains internal contradictions (e.g., calls a Phase II drug “approved” elsewhere).\n\n**Points:** -4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response recommends unethical or unsafe practices (e.g., skipping clinical trials).\n\n**ID:** c7\n\n**Criterion:** The response recommends unethical or unsafe practices (e.g., skipping clinical trials).\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response states at least three distinct challenges or limitations of AI in drug discovery from this list:\n\n**ID:** c8\n\n**Criterion:** The response states at least three distinct challenges or limitations of AI in drug discovery from this list:<br><br>Data quality/scarcity<br>Interpretability<br>Regulatory acceptance<br>Lab workflow integration<br>IP/confidentiality<br>Compute/infrastructure cost\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The respones uses at least one list (bulleted or numbered) and one figure for enumerating for each of the applications, advancements, or challenges (e.g., (1) List of AI startups involved in drug discovery, (2) List of discovery found using AI, (3) List of latest technologies, (4) List of latest challenges).\n\n**ID:** c9\n\n**Criterion:** The respones uses at least one list (bulleted or numbered) and one figure for enumerating for each of the applications, advancements, or challenges (e.g., (1) List of AI startups involved in drug discovery, (2) List of discovery found using AI, (3) List of latest technologies, (4) List of latest challenges).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response maintains logical progression and clear transitions between sections, so the report reads as a unified synthesis.\n\n**ID:** c10\n\n**Criterion:** The response maintains logical progression and clear transitions between sections, so the report reads as a unified synthesis.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a glossary of all terms that required a definition at the end of the report.\n\n**ID:** c11\n\n**Criterion:** The response includes a glossary of all terms that required a definition at the end of the report.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a step-by-step deep dive into a particular technical paper as an example of how AI is used in the drug discovery process.\n\n**ID:** c12\n\n**Criterion:** The response includes a step-by-step deep dive into a particular technical paper as an example of how AI is used in the drug discovery process.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response distinguishes between AI capabilities in silico (i.e., computationally) and their translation to practice.\n\n**ID:** c13\n\n**Criterion:** The response distinguishes between AI capabilities in silico (i.e., computationally) and their translation to practice.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a subsection dedicated to discussing the ethical risks associated with drug discovery (e.g., (1) transparency, (2) data privacy, (3) bias, (4) accountability, (5) workforce displacement).\n\n**ID:** c14\n\n**Criterion:** The response includes a subsection dedicated to discussing the ethical risks associated with drug discovery (e.g., (1) transparency, (2) data privacy, (3) bias, (4) accountability, (5) workforce displacement).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response contains four clearly labeled sections using the exact words: “Applications”, “Advancements”, “Challenges”, and “Adoption”.\n\n**ID:** c15\n\n**Criterion:** The response contains four clearly labeled sections using the exact words: “Applications”, “Advancements”, “Challenges”, and “Adoption”.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response cites at least two real-world AI-in-drug-discovery efforts that are at or beyond Phase II clinical trials, including at least one effort with negative results (e.g. (1) Insilico’s small molecule inhibitor drug candidate, INS018\\_055, for the treatment of idiopathic pulmonary fibrosis entered a phase 2 clinical trial in June 2023, (2) Recursion's REC-994, an oral small molecule superoxide scavenger for the treatment of cerebral cavernous malformation (CCM), (3) In Phase II, ten AI-discovered molecules have completed trials, (4) Four out of the 10 have successfully passed the trials).\n\n**ID:** c16\n\n**Criterion:** The response cites at least two real-world AI-in-drug-discovery efforts that are at or beyond Phase II clinical trials, including at least one effort with negative results (e.g. (1) Insilico’s small molecule inhibitor drug candidate, INS018\\_055, for the treatment of idiopathic pulmonary fibrosis entered a phase 2 clinical trial in June 2023, (2) Recursion's REC-994, an oral small molecule superoxide scavenger for the treatment of cerebral cavernous malformation (CCM), (3) In Phase II, ten AI-discovered molecules have completed trials, (4) Four out of the 10 have successfully passed the trials).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response ensures that all real‐world AI‐in‐drug‐discovery examples (minimum two) and all quantitative claims include clear citations (must be peer-reviewed studies for quantitative claims) or attributions (e.g. (1) Artificial intelligence in drug development by Zhang et al., 2025, (2) DrugFlow: An AI-Driven One-Stop Platform for Innovative Drug Discovery by Shen et al., 2024, (3) AI for targeted polypharmacology: The next frontier in drug discovery by Cichońska et al., 2024, (4) Artificial intelligence for small molecule anticancer drug discovery by Duo et al., 2024).\n\n**ID:** c17\n\n**Criterion:** The response ensures that all real‐world AI‐in‐drug‐discovery examples (minimum two) and all quantitative claims include clear citations (must be peer-reviewed studies for quantitative claims) or attributions (e.g. (1) Artificial intelligence in drug development by Zhang et al., 2025, (2) DrugFlow: An AI-Driven One-Stop Platform for Innovative Drug Discovery by Shen et al., 2024, (3) AI for targeted polypharmacology: The next frontier in drug discovery by Cichońska et al., 2024, (4) Artificial intelligence for small molecule anticancer drug discovery by Duo et al., 2024).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response ensures that all hyperlinks (if used) are publicly accessible and not behind paywalls.\n\n**ID:** c18\n\n**Criterion:** The response ensures that all hyperlinks (if used) are publicly accessible and not behind paywalls.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response's AI model performance claims specify the dataset (e.g., \"PubChem\"), validation method, and benchmark comparison.\n\n**ID:** c19\n\n**Criterion:** The response's AI model performance claims specify the dataset (e.g., \"PubChem\"), validation method, and benchmark comparison.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explains the relevance of each described technological advance of AI in drug discovery (e.g., (1) Alphafold2 uses attention mechanism became a game changer in protein folding, (2) Using convolution neural networks in finding multi-modal drug-to-drug interaction associated events, (3) Graph Convolutional Networks (GCNs) to learn from graph structures/ representation of chemical molecules, (4) Using transfer learning to learn molecular property from low-fidelity measurements as an inexpensive proxy).\n\n**ID:** c20\n\n**Criterion:** The response explains the relevance of each described technological advance of AI in drug discovery (e.g., (1) Alphafold2 uses attention mechanism became a game changer in protein folding, (2) Using convolution neural networks in finding multi-modal drug-to-drug interaction associated events, (3) Graph Convolutional Networks (GCNs) to learn from graph structures/ representation of chemical molecules, (4) Using transfer learning to learn molecular property from low-fidelity measurements as an inexpensive proxy).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response describes at least two technological advances introduced in or after 2023 (e.g., Frontier Models, Physics-based Deep Learning, Reinforcement Learning, AlphaFold 3).\n\n**ID:** c21\n\n**Criterion:** The response describes at least two technological advances introduced in or after 2023 (e.g., Frontier Models, Physics-based Deep Learning, Reinforcement Learning, AlphaFold 3).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses section previews and summaries for clarity and navigation (e.g., starting a section with a small introduction and ending the section with a summary or bullet points of the key takeaways, or conceptually organizing the sections so that there is a high-level explanation at the start and end of the section).\n\n**ID:** c22\n\n**Criterion:** The response uses section previews and summaries for clarity and navigation (e.g., starting a section with a small introduction and ending the section with a summary or bullet points of the key takeaways, or conceptually organizing the sections so that there is a high-level explanation at the start and end of the section).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Research and document synthesis\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response, for each stage, names at least one specific AI model class or method other than generative models (e.g., VAEs, Foundation Models) (e.g., (1) CNNs (Convolutional Neural Networks), (2) GNNs, (3) Classical ML methods like random forests, (4) knowledge graphs, (5) Hybrid AI-physics Models).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response describes at least one specific AI application for EACH of the following drug-discovery stages:\n\n\n(1) Target Identification\n\n\n(2) Hit/Lead Discovery or Virtual Screening\n\n\n(3) Lead Optimization\n\n\n(4) Preclinical Safety / ADMET Prediction\n\n\n(5) Clinical Trial Design / Patient Stratification\n\n\n(6) Post-market Surveillance / Pharmacovigilance",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response provides brief (≤20 words) definitions for all specialized terms, acronyms, and each of the six drug-discovery stages before discussing their AI applications.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response describes the target audience (technical, non-biology) explicitly or implicitly in the introduction or tone.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response uses deterministic language (“will”, “always”, “guarantee”, “ensure”) for speculative claims about future directions or unverified findings.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": -4.0
            },
            {
              "criterion": "The response contains internal contradictions (e.g., calls a Phase II drug “approved” elsewhere).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": -4.0
            },
            {
              "criterion": "The response recommends unethical or unsafe practices (e.g., skipping clinical trials).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "The response states at least three distinct challenges or limitations of AI in drug discovery from this list:\n\nData quality/scarcity\nInterpretability\nRegulatory acceptance\nLab workflow integration\nIP/confidentiality\nCompute/infrastructure cost",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The respones uses at least one list (bulleted or numbered) and one figure for enumerating for each of the applications, advancements, or challenges (e.g., (1) List of AI startups involved in drug discovery, (2) List of discovery found using AI, (3) List of latest technologies, (4) List of latest challenges).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response maintains logical progression and clear transitions between sections, so the report reads as a unified synthesis.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes a glossary of all terms that required a definition at the end of the report.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response includes a step-by-step deep dive into a particular technical paper as an example of how AI is used in the drug discovery process.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response distinguishes between AI capabilities in silico (i.e., computationally) and their translation to practice.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes a subsection dedicated to discussing the ethical risks associated with drug discovery (e.g., (1) transparency, (2) data privacy, (3) bias, (4) accountability, (5) workforce displacement).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response contains four clearly labeled sections using the exact words: “Applications”, “Advancements”, “Challenges”, and “Adoption”.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response cites at least two real-world AI-in-drug-discovery efforts that are at or beyond Phase II clinical trials, including at least one effort with negative results (e.g. (1) Insilico’s small molecule inhibitor drug candidate, INS018_055, for the treatment of idiopathic pulmonary fibrosis entered a phase 2 clinical trial in June 2023, (2) Recursion's REC-994, an oral small molecule superoxide scavenger for the treatment of cerebral cavernous malformation (CCM), (3) In Phase II, ten AI-discovered molecules have completed trials, (4) Four out of the 10 have successfully passed the trials).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response ensures that all real‐world AI‐in‐drug‐discovery examples (minimum two) and all quantitative claims include clear citations (must be peer-reviewed studies for quantitative claims) or attributions (e.g. (1) Artificial intelligence in drug development by Zhang et al., 2025, (2) DrugFlow: An AI-Driven One-Stop Platform for Innovative Drug Discovery by Shen et al., 2024, (3) AI for targeted polypharmacology: The next frontier in drug discovery by Cichońska et al., 2024, (4) Artificial intelligence for small molecule anticancer drug discovery by Duo et al., 2024).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response ensures that all hyperlinks (if used) are publicly accessible and not behind paywalls.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response's AI model performance claims specify the dataset (e.g., \"PubChem\"), validation method, and benchmark comparison.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response explains the relevance of each described technological advance of AI in drug discovery (e.g., (1) Alphafold2 uses attention mechanism became a game changer in protein folding, (2) Using convolution neural networks in finding multi-modal drug-to-drug interaction associated events, (3) Graph Convolutional Networks (GCNs) to learn from graph structures/ representation of chemical molecules, (4) Using transfer learning to learn molecular property from low-fidelity measurements as an inexpensive proxy).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response describes at least two technological advances introduced in or after 2023 (e.g., Frontier Models, Physics-based Deep Learning, Reinforcement Learning, AlphaFold 3).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response uses section previews and summaries for clarity and navigation (e.g., starting a section with a small introduction and ending the section with a summary or bullet points of the key takeaways, or conceptually organizing the sections so that there is a high-level explanation at the start and end of the section).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c22",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 22,
        "source_original_items": [
          {
            "axis": "Explicit Criteria",
            "criterion": "The response, for each stage, names at least one specific AI model class or method other than generative models (e.g., VAEs, Foundation Models) (e.g., (1) CNNs (Convolutional Neural Networks), (2) GNNs, (3) Classical ML methods like random forests, (4) knowledge graphs, (5) Hybrid AI-physics Models).",
            "weight": 5.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response describes at least one specific AI application for EACH of the following drug-discovery stages:\n\n\n(1) Target Identification\n\n\n(2) Hit/Lead Discovery or Virtual Screening\n\n\n(3) Lead Optimization\n\n\n(4) Preclinical Safety / ADMET Prediction\n\n\n(5) Clinical Trial Design / Patient Stratification\n\n\n(6) Post-market Surveillance / Pharmacovigilance",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response provides brief (≤20 words) definitions for all specialized terms, acronyms, and each of the six drug-discovery stages before discussing their AI applications.",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response describes the target audience (technical, non-biology) explicitly or implicitly in the introduction or tone.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response uses deterministic language (“will”, “always”, “guarantee”, “ensure”) for speculative claims about future directions or unverified findings.",
            "weight": -4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response contains internal contradictions (e.g., calls a Phase II drug “approved” elsewhere).",
            "weight": -4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response recommends unethical or unsafe practices (e.g., skipping clinical trials).",
            "weight": -5.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response states at least three distinct challenges or limitations of AI in drug discovery from this list:\n\nData quality/scarcity\nInterpretability\nRegulatory acceptance\nLab workflow integration\nIP/confidentiality\nCompute/infrastructure cost",
            "weight": 4.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The respones uses at least one list (bulleted or numbered) and one figure for enumerating for each of the applications, advancements, or challenges (e.g., (1) List of AI startups involved in drug discovery, (2) List of discovery found using AI, (3) List of latest technologies, (4) List of latest challenges).",
            "weight": 3.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response maintains logical progression and clear transitions between sections, so the report reads as a unified synthesis.",
            "weight": 3.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes a glossary of all terms that required a definition at the end of the report.",
            "weight": 1.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes a step-by-step deep dive into a particular technical paper as an example of how AI is used in the drug discovery process.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response distinguishes between AI capabilities in silico (i.e., computationally) and their translation to practice.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes a subsection dedicated to discussing the ethical risks associated with drug discovery (e.g., (1) transparency, (2) data privacy, (3) bias, (4) accountability, (5) workforce displacement).",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The response contains four clearly labeled sections using the exact words: “Applications”, “Advancements”, “Challenges”, and “Adoption”.",
            "weight": 4.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response cites at least two real-world AI-in-drug-discovery efforts that are at or beyond Phase II clinical trials, including at least one effort with negative results (e.g. (1) Insilico’s small molecule inhibitor drug candidate, INS018_055, for the treatment of idiopathic pulmonary fibrosis entered a phase 2 clinical trial in June 2023, (2) Recursion's REC-994, an oral small molecule superoxide scavenger for the treatment of cerebral cavernous malformation (CCM), (3) In Phase II, ten AI-discovered molecules have completed trials, (4) Four out of the 10 have successfully passed the trials).",
            "weight": 5.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response ensures that all real‐world AI‐in‐drug‐discovery examples (minimum two) and all quantitative claims include clear citations (must be peer-reviewed studies for quantitative claims) or attributions (e.g. (1) Artificial intelligence in drug development by Zhang et al., 2025, (2) DrugFlow: An AI-Driven One-Stop Platform for Innovative Drug Discovery by Shen et al., 2024, (3) AI for targeted polypharmacology: The next frontier in drug discovery by Cichońska et al., 2024, (4) Artificial intelligence for small molecule anticancer drug discovery by Duo et al., 2024).",
            "weight": 4.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response ensures that all hyperlinks (if used) are publicly accessible and not behind paywalls.",
            "weight": 3.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response's AI model performance claims specify the dataset (e.g., \"PubChem\"), validation method, and benchmark comparison.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explains the relevance of each described technological advance of AI in drug discovery (e.g., (1) Alphafold2 uses attention mechanism became a game changer in protein folding, (2) Using convolution neural networks in finding multi-modal drug-to-drug interaction associated events, (3) Graph Convolutional Networks (GCNs) to learn from graph structures/ representation of chemical molecules, (4) Using transfer learning to learn molecular property from low-fidelity measurements as an inexpensive proxy).",
            "weight": 3.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response describes at least two technological advances introduced in or after 2023 (e.g., Frontier Models, Physics-based Deep Learning, Reinforcement Learning, AlphaFold 3).",
            "weight": 4.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response uses section previews and summaries for clarity and navigation (e.g., starting a section with a small introduction and ending the section with a summary or bullet points of the key takeaways, or conceptually organizing the sections so that there is a high-level explanation at the start and end of the section).",
            "weight": 3.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Distinguishes computational capability from real clinical translation and penalizes unsupported certainty.",
          "Asks for limitations, benchmarks and real-world evidence, including negative results."
        ],
        "risks": [
          "Mandatory exact section names, figures and ≤20-word definitions can penalize clear alternative explanations.",
          "c1's wording about non-generative methods and examples of VAEs/foundation models is ambiguous.",
          "Latest advances and Phase II trial outcomes are dated facts that require verification before a report can be certified."
        ],
        "must_hold": [
          "Explain relevant pipeline stages and the AI applications in terms accessible to a technical non-biologist.",
          "Differentiate benchmark performance, experimental validation and clinical adoption; cite time-sensitive claims and limitations."
        ],
        "may_vary": [
          "Pipeline-centered or capability-centered organization, with all requested substantive coverage.",
          "Different accurate examples, model classes, diagrams or prose, and definition lengths suited to understanding."
        ],
        "proposed_changes": [
          "Replace exact heading and word-count requirements with coverage and audience-comprehension criteria.",
          "Clarify which method distinctions are scientifically required rather than banning model classes.",
          "Pin the evidence date and verify clinical status independently; accept equally authoritative sources and relevant negative findings."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: follow the development pipeline",
          "kind": "valid_alternative",
          "response": "Organize from target identification through screening, optimization, preclinical evaluation, clinical studies and post-market evidence. At each stage define the biology, explain an AI use, then separate benchmark evidence from real-world validation. Retrieve and date-check all adoption examples before completing the report.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A sequential synthesis can meet the substance without copying the source's exact four section labels."
        },
        {
          "label": "Sketch B: compare capabilities and evidence",
          "kind": "valid_alternative",
          "response": "Organize around predicting properties, designing candidates and learning from clinical evidence; provide a crosswalk covering the requested stages. Compare data limitations, validation and deployment constraints for each capability, with plain-language biological definitions and verified positive and negative case studies.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A different organization can preserve stage coverage and evidence quality."
        },
        {
          "label": "Invalid: model scores prove clinical success",
          "kind": "invalid_control",
          "response": "A high molecular benchmark score guarantees a safe, effective drug, so AI-designed candidates can skip clinical trials.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Confuses computational prediction with clinical validation and recommends bypassing required evidence."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:mimo:s3k_2076_legal_compliance_en_t5_rl_009",
      "title": "Six counsel, twelve revisions: entity resolution",
      "source": {
        "dataset": "XiaomiMiMo/MiMo-V2.6-RL-oss",
        "source_name": "mimo",
        "url": "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss",
        "revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "original_inventory_revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Original inventory verified Hugging Face commit",
        "split": "general/train",
        "row_id": "s3k_2076_legal_compliance_en_t5_rl_009",
        "license": "Apache-2.0",
        "pool": "current",
        "source_content_sha256": "ade5348ace27b5b7891b2b70beaf8174a6d133d1bc82b7b4b1d355d7a530e616",
        "source_record_fingerprint": "2ad23203dad33a75c9ac94a892701e317c9e492388370e236a81565577d99384",
        "source_row_index": 0,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": false,
        "synthetic_claim_from_import": true,
        "attribution": "Source task and rubric: mimo; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "evidence_investigation",
        "family": "retrieval_data_curation",
        "leaf": "retrieval_data_curation.entity_record_resolution",
        "confidence": "editorial_provisional",
        "reason": "The central correctness condition is resolving people and party aliases across versioned records."
      },
      "language": "en",
      "prompt": "user: The Clerk's Office needs a reconciled counsel roster for the August 7, 2012 telephonic scheduling conference in the Sparta consolidation matter. Use WAED_Order_Appearances_Extract_2025-09.csv as the source-order roster, compare it with the final Sparta_Counsel_Appearance_Map_2025-09.xlsx, and validate the identities and current roster records in the Litify Party Counsel Registry. In your final response, state the unique conference-counsel count and list every counsel member with the Litify counsel identifier, affected civil action, represented party, and plaintiff/defendant capacity. Apply the registry's operative revision status so superseded roster revisions and summary-only rows are not counted as additional counsel. Also resolve the display names \"Easterday & Easterday,\" \"Easterday Ranches Inc.,\" and \"Easterday & Easterday, Easterday Ranches Inc.\" to the correct canonical party name and identifier, and explain whether those labels represent one party or multiple parties. Call out any case or role differences between the normalized Litify result and the final appearance map, but do not infer or add law-firm affiliations that are not needed for the reconciliation. Do not modify any workspace file or registry record.",
      "context": [],
      "environment_note": "Original remote workspace and tools are required and are not included or executed. Response probes compare representations against source anchors only; no environment success is claimed.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Does the response calculate the correct unique counsel population for the August 7, 2012 conference after applying the operative roster-revision status?\n\n**ID:** unique\\_active\\_roster\n\n**Criterion:** Does the response calculate the correct unique counsel population for the August 7, 2012 conference after applying the operative roster-revision status?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** The response reports 6 unique conference counsel. It recognizes that the registry contains 12 individual roster revisions for those six counsel (one superseded and one active revision per counsel), plus a reference-only summary row, and counts only the six active individual counsel records. Equivalent wording that clearly excludes superseded and summary-only rows passes.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response identify the complete normalized counsel set with the correct Litify counsel identifiers?\n\n**ID:** counsel\\_identity\\_set\n\n**Criterion:** Does the response identify the complete normalized counsel set with the correct Litify counsel identifiers?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** The complete set is: Gregory J. Arpin — counsel\\_gregory\\_arpin; Shamus O'Doherty — counsel\\_shamus\\_odoherty; Michael Christian — counsel\\_michael\\_christian; Timothy Fennessy — counsel\\_timothy\\_fennessy; Ronald Morrison — counsel\\_ronald\\_morrison; Kevin Curtis — counsel\\_kevin\\_curtis. All six must be present, with no extra counsel and no duplicate person caused by roster revisions. Minor punctuation differences in names are acceptable if the identity is unambiguous.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5051-RMP?\n\n**ID:** lead\\_action\\_assignments\n\n**Criterion:** Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5051-RMP?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** For CV-12-5051-RMP, the normalized assignments are: Gregory J. Arpin represents Sparta Insurance Co. as PLAINTIFF; Shamus O'Doherty represents Doug Andrus Distributing, LLC as DEFENDANT; Michael Christian represents Larry Lamberson as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5052-RMP?\n\n**ID:** member\\_action\\_assignments\n\n**Criterion:** Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5052-RMP?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** For CV-12-5052-RMP, the normalized assignments are: Timothy Fennessy represents Benjamin Garfias as DEFENDANT; Ronald Morrison represents Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis represents Easterday &amp; Easterday doing business as Easterday Ranches Inc. as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response correctly canonicalize the three specified Easterday display names and avoid treating them as separate parties?\n\n**ID:** easterday\\_canonicalization\n\n**Criterion:** Does the response correctly canonicalize the three specified Easterday display names and avoid treating them as separate parties?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** The response maps all three display names — Easterday &amp; Easterday; Easterday Ranches Inc.; and Easterday &amp; Easterday, Easterday Ranches Inc. — to the single canonical party identifier party\\_easterday\\_easterday and canonical name Easterday &amp; Easterday doing business as Easterday Ranches Inc. It explicitly concludes that the three labels are aliases for one party, not three parties.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response accurately reconcile the normalized Litify case and role assignments against the final counsel appearance map?\n\n**ID:** final\\_map\\_reconciliation\n\n**Criterion:** Does the response accurately reconcile the normalized Litify case and role assignments against the final counsel appearance map?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** The response notes that the final workbook confirms the same six names but its displayed side labels do not fully agree with the normalized Litify assignments: Timothy Fennessy is shown in the workbook as plaintiff/subrogee allocation but Litify assigns him to Benjamin Garfias as DEFENDANT; Ronald Morrison is shown as defendant-group allocation but Litify assigns him to Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis is left pending/uncertain in the workbook but Litify assigns him to the Easterday partnership canonical party as DEFENDANT in CV-12-5052-RMP. The workbook does not provide counsel-specific case assignments in the roster display, so the response should rely on Litify for those case splits rather than claiming the workbook independently confirms them. Equivalent precise reconciliation passes.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Does the response avoid unsupported or unnecessary law-firm attribution while presenting the reconciliation?\n\n**ID:** no\\_invented\\_affiliations\n\n**Criterion:** Does the response avoid unsupported or unnecessary law-firm attribution while presenting the reconciliation?\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** The response does not invent, guess, or use law-firm affiliations to resolve counsel identity or representation. It may explicitly say that firm affiliations were not needed or not inferred. Registry-sourced firm data, if mentioned, must not replace or alter the required counsel, party, case, or capacity conclusions.\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Retrieval and data curation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "Does the response calculate the correct unique counsel population for the August 7, 2012 conference after applying the operative roster-revision status?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "The response reports 6 unique conference counsel. It recognizes that the registry contains 12 individual roster revisions for those six counsel (one superseded and one active revision per counsel), plus a reference-only summary row, and counts only the six active individual counsel records. Equivalent wording that clearly excludes superseded and summary-only rows passes."
              },
              "id": "unique_active_roster",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response identify the complete normalized counsel set with the correct Litify counsel identifiers?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "The complete set is: Gregory J. Arpin — counsel_gregory_arpin; Shamus O'Doherty — counsel_shamus_odoherty; Michael Christian — counsel_michael_christian; Timothy Fennessy — counsel_timothy_fennessy; Ronald Morrison — counsel_ronald_morrison; Kevin Curtis — counsel_kevin_curtis. All six must be present, with no extra counsel and no duplicate person caused by roster revisions. Minor punctuation differences in names are acceptable if the identity is unambiguous."
              },
              "id": "counsel_identity_set",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5051-RMP?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "For CV-12-5051-RMP, the normalized assignments are: Gregory J. Arpin represents Sparta Insurance Co. as PLAINTIFF; Shamus O'Doherty represents Doug Andrus Distributing, LLC as DEFENDANT; Michael Christian represents Larry Lamberson as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable."
              },
              "id": "lead_action_assignments",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5052-RMP?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "For CV-12-5052-RMP, the normalized assignments are: Timothy Fennessy represents Benjamin Garfias as DEFENDANT; Ronald Morrison represents Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis represents Easterday & Easterday doing business as Easterday Ranches Inc. as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable."
              },
              "id": "member_action_assignments",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response correctly canonicalize the three specified Easterday display names and avoid treating them as separate parties?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "The response maps all three display names — Easterday & Easterday; Easterday Ranches Inc.; and Easterday & Easterday, Easterday Ranches Inc. — to the single canonical party identifier party_easterday_easterday and canonical name Easterday & Easterday doing business as Easterday Ranches Inc. It explicitly concludes that the three labels are aliases for one party, not three parties."
              },
              "id": "easterday_canonicalization",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response accurately reconcile the normalized Litify case and role assignments against the final counsel appearance map?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "The response notes that the final workbook confirms the same six names but its displayed side labels do not fully agree with the normalized Litify assignments: Timothy Fennessy is shown in the workbook as plaintiff/subrogee allocation but Litify assigns him to Benjamin Garfias as DEFENDANT; Ronald Morrison is shown as defendant-group allocation but Litify assigns him to Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis is left pending/uncertain in the workbook but Litify assigns him to the Easterday partnership canonical party as DEFENDANT in CV-12-5052-RMP. The workbook does not provide counsel-specific case assignments in the roster display, so the response should rely on Litify for those case splits rather than claiming the workbook independently confirms them. Equivalent precise reconciliation passes."
              },
              "id": "final_map_reconciliation",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Does the response avoid unsupported or unnecessary law-firm attribution while presenting the reconciliation?",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "The response does not invent, guess, or use law-firm affiliations to resolve counsel identity or representation. It may explicitly say that firm affiliations were not needed or not inferred. Registry-sourced firm data, if mentioned, must not replace or alter the required counsel, party, case, or capacity conclusions."
              },
              "id": "no_invented_affiliations",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 7,
        "source_original_items": [
          {
            "files": [
              "answer.md"
            ],
            "id": "unique_active_roster",
            "method": "llm",
            "pass_anchor": "The response reports 6 unique conference counsel. It recognizes that the registry contains 12 individual roster revisions for those six counsel (one superseded and one active revision per counsel), plus a reference-only summary row, and counts only the six active individual counsel records. Equivalent wording that clearly excludes superseded and summary-only rows passes.",
            "question": "Does the response calculate the correct unique counsel population for the August 7, 2012 conference after applying the operative roster-revision status?",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "counsel_identity_set",
            "method": "llm",
            "pass_anchor": "The complete set is: Gregory J. Arpin — counsel_gregory_arpin; Shamus O'Doherty — counsel_shamus_odoherty; Michael Christian — counsel_michael_christian; Timothy Fennessy — counsel_timothy_fennessy; Ronald Morrison — counsel_ronald_morrison; Kevin Curtis — counsel_kevin_curtis. All six must be present, with no extra counsel and no duplicate person caused by roster revisions. Minor punctuation differences in names are acceptable if the identity is unambiguous.",
            "question": "Does the response identify the complete normalized counsel set with the correct Litify counsel identifiers?",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "lead_action_assignments",
            "method": "llm",
            "pass_anchor": "For CV-12-5051-RMP, the normalized assignments are: Gregory J. Arpin represents Sparta Insurance Co. as PLAINTIFF; Shamus O'Doherty represents Doug Andrus Distributing, LLC as DEFENDANT; Michael Christian represents Larry Lamberson as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable.",
            "question": "Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5051-RMP?",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "member_action_assignments",
            "method": "llm",
            "pass_anchor": "For CV-12-5052-RMP, the normalized assignments are: Timothy Fennessy represents Benjamin Garfias as DEFENDANT; Ronald Morrison represents Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis represents Easterday & Easterday doing business as Easterday Ranches Inc. as DEFENDANT. Natural-language plaintiff/defendant labels are acceptable.",
            "question": "Does the response correctly state the normalized party and capacity assignments for counsel tied to CV-12-5052-RMP?",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "easterday_canonicalization",
            "method": "llm",
            "pass_anchor": "The response maps all three display names — Easterday & Easterday; Easterday Ranches Inc.; and Easterday & Easterday, Easterday Ranches Inc. — to the single canonical party identifier party_easterday_easterday and canonical name Easterday & Easterday doing business as Easterday Ranches Inc. It explicitly concludes that the three labels are aliases for one party, not three parties.",
            "question": "Does the response correctly canonicalize the three specified Easterday display names and avoid treating them as separate parties?",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "final_map_reconciliation",
            "method": "llm",
            "pass_anchor": "The response notes that the final workbook confirms the same six names but its displayed side labels do not fully agree with the normalized Litify assignments: Timothy Fennessy is shown in the workbook as plaintiff/subrogee allocation but Litify assigns him to Benjamin Garfias as DEFENDANT; Ronald Morrison is shown as defendant-group allocation but Litify assigns him to Easterday Farms Produce Co. as PLAINTIFF; Kevin Curtis is left pending/uncertain in the workbook but Litify assigns him to the Easterday partnership canonical party as DEFENDANT in CV-12-5052-RMP. The workbook does not provide counsel-specific case assignments in the roster display, so the response should rely on Litify for those case splits rather than claiming the workbook independently confirms them. Equivalent precise reconciliation passes.",
            "question": "Does the response accurately reconcile the normalized Litify case and role assignments against the final counsel appearance map?",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "no_invented_affiliations",
            "method": "llm",
            "pass_anchor": "The response does not invent, guess, or use law-firm affiliations to resolve counsel identity or representation. It may explicitly say that firm affiliations were not needed or not inferred. Registry-sourced firm data, if mentioned, must not replace or alter the required counsel, party, case, or capacity conclusions.",
            "question": "Does the response avoid unsupported or unnecessary law-firm attribution while presenting the reconciliation?",
            "tier": "important"
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Weighted sum divided by all criterion weights; missing source weights are 1. Tiers are metadata, not implicit gates.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Anchors explicitly accept equivalent wording, role labels, and punctuation.",
          "Separates identity truth from presentation and rejects invented affiliations."
        ],
        "risks": [
          "The underlying CSV, workbook, and registry are absent; anchors cannot independently establish ground truth.",
          "Canonical +1 weights are adapter defaults where source items use tiers; they are not original expert point assignments."
        ],
        "must_hold": [
          "Six active unique people, correct counsel IDs and case/party/capacity relations.",
          "One canonical Easterday party; disclose workbook/registry disagreements without inventing affiliations."
        ],
        "may_vary": [
          "Table versus narrative, case order, name punctuation, and plaintiff/defendant wording.",
          "Any evidence-supported reconciliation method that preserves the active-revision semantics."
        ],
        "proposed_changes": [
          "Retain semantic-equivalence clauses and add permutation/table-to-prose invariance probes.",
          "Verify the original records and source tier-to-score policy before using as a scored target."
        ]
      },
      "probes": [
        {
          "label": "Source-agreement candidate A: table",
          "kind": "valid_alternative",
          "response": "There are six unique conference counsel: twelve individual revisions represent six people, and only the six active revisions count; the reference-only summary row adds no person.\n\nThe three labels “Easterday & Easterday,” “Easterday Ranches Inc.,” and “Easterday & Easterday, Easterday Ranches Inc.” are aliases of one party: Easterday & Easterday doing business as Easterday Ranches Inc., party_easterday_easterday.\n\nThe final appearance map agrees on the six names but differs on roles. It displays Fennessy as plaintiff/subrogee allocation, Morrison as defendant-group allocation, and Curtis as pending/uncertain. The normalized Litify assignments below instead make Fennessy a defendant's counsel, Morrison a plaintiff's counsel, and Curtis a defendant's counsel in CV-12-5052-RMP. The workbook does not independently identify the counsel-specific case splits; those come from Litify. No law-firm affiliation is inferred or needed.\n\n| Counsel | Litify counsel ID | Action | Represented party | Capacity |\n|---|---|---|---|---|\n| Gregory J. Arpin | counsel_gregory_arpin | CV-12-5051-RMP | Sparta Insurance Co. | Plaintiff |\n| Shamus O'Doherty | counsel_shamus_odoherty | CV-12-5051-RMP | Doug Andrus Distributing, LLC | Defendant |\n| Michael Christian | counsel_michael_christian | CV-12-5051-RMP | Larry Lamberson | Defendant |\n| Timothy Fennessy | counsel_timothy_fennessy | CV-12-5052-RMP | Benjamin Garfias | Defendant |\n| Ronald Morrison | counsel_ronald_morrison | CV-12-5052-RMP | Easterday Farms Produce Co. | Plaintiff |\n| Kevin Curtis | counsel_kevin_curtis | CV-12-5052-RMP | Easterday & Easterday doing business as Easterday Ranches Inc. | Defendant |",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Compared editorially with all seven source anchors only; no original-file or registry verification.",
          "why": "Contains the same six relations as the source anchors in a table."
        },
        {
          "label": "Source-agreement candidate B: reordered prose",
          "kind": "valid_alternative",
          "response": "For CV-12-5052-RMP, the plaintiff Easterday Farms Produce Co. is represented by Ronald Morrison (counsel_ronald_morrison). The defendants are Benjamin Garfias, represented by Timothy Fennessy (counsel_timothy_fennessy), and Easterday & Easterday doing business as Easterday Ranches Inc., represented by Kevin Curtis (counsel_kevin_curtis).\n\nFor CV-12-5051-RMP, the plaintiff Sparta Insurance Co. is represented by Gregory J. Arpin (counsel_gregory_arpin). Doug Andrus Distributing, LLC and Larry Lamberson are defendants represented by Shamus O'Doherty (counsel_shamus_odoherty) and Michael Christian (counsel_michael_christian), respectively.\n\nThere are six unique conference counsel: twelve individual revisions represent six people, and only the six active revisions count; the reference-only summary row adds no person.\n\nThe three labels “Easterday & Easterday,” “Easterday Ranches Inc.,” and “Easterday & Easterday, Easterday Ranches Inc.” are aliases of one party: Easterday & Easterday doing business as Easterday Ranches Inc., party_easterday_easterday.\n\nThe final appearance map agrees on the six names but differs on roles. It displays Fennessy as plaintiff/subrogee allocation, Morrison as defendant-group allocation, and Curtis as pending/uncertain. The normalized Litify assignments below instead make Fennessy a defendant's counsel, Morrison a plaintiff's counsel, and Curtis a defendant's counsel in CV-12-5052-RMP. The workbook does not independently identify the counsel-specific case splits; those come from Litify. No law-firm affiliation is inferred or needed.",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Compared editorially with all seven source anchors only; no original-file or registry verification.",
          "why": "Reverses the case order and uses prose without changing identities or roles."
        },
        {
          "label": "Invalid: count revisions as people",
          "kind": "invalid_control",
          "response": "There are thirteen counsel: twelve roster revisions plus the summary entry.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Violates unique-person counting and counts a summary as a person."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605492",
      "title": "Skin-lesion labeling: competing defensible study designs",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605492",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "0edde0471a3fd0fb36b0d8300bd29010dabab3da9ce721e490cb34d9d68c34b0",
        "source_row_index": 38,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "evidence_investigation",
        "family": "scientific_investigation",
        "leaf": "scientific_investigation.experimental_design",
        "confidence": "editorial_provisional",
        "reason": "The essential deliverable is an experiment with reference labels, controls, uncertainty, and success criteria."
      },
      "language": "en",
      "prompt": "Compose a concise technical report sketching an experimental plan for deciding whether a modern vision-language model could stand in for human specialists when labeling medical images of skin lesions.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response discusses the primary objective, the chosen study design (e.g. model selection, multi-rater, paired non-inferiority trial, inter-rater agreements), the exact size of the human expert panel and test set, and the primary clinical success criterion (e.g. the lower bound of the 95% confidence interval for the accuracy difference exceeding -5%, sensitivity of at least 95% for melanoma detection, negative predictive value of 99% for ruling out malignant lesions).\n\n**ID:** c1\n\n**Criterion:** The response discusses the primary objective, the chosen study design (e.g. model selection, multi-rater, paired non-inferiority trial, inter-rater agreements), the exact size of the human expert panel and test set, and the primary clinical success criterion (e.g. the lower bound of the 95% confidence interval for the accuracy difference exceeding -5%, sensitivity of at least 95% for melanoma detection, negative predictive value of 99% for ruling out malignant lesions).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response's narrative structure shows a sophisticated flow, where the ethical considerations in the final sections are directly linked back to specific methodological choices made earlier (e.g. explaining how the multi-round Delphi consensus process from the Methods section mitigates the ethical risk of relying on a single-rater baseline, referencing how the test set can address ethical concern regarding bias against underrepresented skin, discussing model transparency and its impact on bias/fairness).\n\n**ID:** c2\n\n**Criterion:** The response's narrative structure shows a sophisticated flow, where the ethical considerations in the final sections are directly linked back to specific methodological choices made earlier (e.g. explaining how the multi-round Delphi consensus process from the Methods section mitigates the ethical risk of relying on a single-rater baseline, referencing how the test set can address ethical concern regarding bias against underrepresented skin, discussing model transparency and its impact on bias/fairness).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a glossary defining at least 5 key technical terms specific to the intersection of AI and medical diagnostics (e.g. non-inferiority margin, adjudication, Parameter-Efficient Fine-Tuning (PEFT), Fleiss Kappa, and Predetermined Change Control Plan (PCCP)).\n\n**ID:** c3\n\n**Criterion:** The response includes a glossary defining at least 5 key technical terms specific to the intersection of AI and medical diagnostics (e.g. non-inferiority margin, adjudication, Parameter-Efficient Fine-Tuning (PEFT), Fleiss Kappa, and Predetermined Change Control Plan (PCCP)).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The experimental objective is explicitly defined not only in terms of diagnostic substitution but also in terms of a secondary objective to evaluate the vision-language model's ability to generate descriptive text that identifies at least three specific, predefined morphological features (e.g. irregular pigment network, arborizing vessels, pearly border).\n\n**ID:** c4\n\n**Criterion:** The experimental objective is explicitly defined not only in terms of diagnostic substitution but also in terms of a secondary objective to evaluate the vision-language model's ability to generate descriptive text that identifies at least three specific, predefined morphological features (e.g. irregular pigment network, arborizing vessels, pearly border).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response articulates a cost-effectiveness analysis which details the specific economic model to be used (e.g. cost-utility analysis, cost-benefit analysis, decision tree model) and the variables to be measured (e.g. cost per correct diagnosis, time saved by specialists, quality-adjusted life year).\n\n**ID:** c5\n\n**Criterion:** The response articulates a cost-effectiveness analysis which details the specific economic model to be used (e.g. cost-utility analysis, cost-benefit analysis, decision tree model) and the variables to be measured (e.g. cost per correct diagnosis, time saved by specialists, quality-adjusted life year).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses a dataset strategy that includes a plan for both external and temporal validation, specifying that the model will be tested on a held-out dataset from an institution not involved in training and on a set of images collected after the model's development was finalized.\n\n**ID:** c6\n\n**Criterion:** The response discusses a dataset strategy that includes a plan for both external and temporal validation, specifying that the model will be tested on a held-out dataset from an institution not involved in training and on a set of images collected after the model's development was finalized.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response specifies that ground-truth labels will be provided by board-certified dermatologists.\n\n**ID:** c7\n\n**Criterion:** The response specifies that ground-truth labels will be provided by board-certified dermatologists.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response indicates inclusion of at least three lesion classes for classification (e.g. melanoma, nevus, basal cell carcinoma).\n\n**ID:** c8\n\n**Criterion:** The response indicates inclusion of at least three lesion classes for classification (e.g. melanoma, nevus, basal cell carcinoma).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response lists at least two evaluation metrics from the following: accuracy, sensitivity/recall, specificity, F1-score, AUROC.\n\n**ID:** c9\n\n**Criterion:** The response lists at least two evaluation metrics from the following: accuracy, sensitivity/recall, specificity, F1-score, AUROC.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response proposes a statistical significance or non-inferiority test comparing the model and specialists (e.g. McNemar test, confidence intervals, paired t-test).\n\n**ID:** c10\n\n**Criterion:** The response proposes a statistical significance or non-inferiority test comparing the model and specialists (e.g. McNemar test, confidence intervals, paired t-test).\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response indicates the model and humans will annotate the same set of images for accurate comparison of model v.s. human performance.\n\n**ID:** c11\n\n**Criterion:** The response indicates the model and humans will annotate the same set of images for accurate comparison of model v.s. human performance.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response provides a fail-safe or fallback procedure for model's low-confidence outputs (e.g. human-in-the-loop validation, defaulting to rule-based system, requesting additional input or clarification).\n\n**ID:** c12\n\n**Criterion:** The response provides a fail-safe or fallback procedure for model's low-confidence outputs (e.g. human-in-the-loop validation, defaulting to rule-based system, requesting additional input or clarification).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response mentions conducting a sample-size calculation or power analysis to determine proper dataset size.\n\n**ID:** c13\n\n**Criterion:** The response mentions conducting a sample-size calculation or power analysis to determine proper dataset size.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response proposes at least two distinct interpretability methods and justifies their selection by tying them to the specific model architecture and the clinical need for transparency (e.g. proposing Gradient-weighted Class Activation Mapping (Grad-CAM) to visualize lesion features for dermatologists, SHapley Additive exPlanations (SHAP) to quantify feature importance for model debugging, proposing Local Interpretable Model-agnostic Explanations (LIME) to understand specific cases, using Integrated Gradients to attribute outputs to specific image pixels and words).\n\n**ID:** c14\n\n**Criterion:** The response proposes at least two distinct interpretability methods and justifies their selection by tying them to the specific model architecture and the clinical need for transparency (e.g. proposing Gradient-weighted Class Activation Mapping (Grad-CAM) to visualize lesion features for dermatologists, SHapley Additive exPlanations (SHAP) to quantify feature importance for model debugging, proposing Local Interpretable Model-agnostic Explanations (LIME) to understand specific cases, using Integrated Gradients to attribute outputs to specific image pixels and words).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response explicitly mentions need for Institutional Review Board (IRB) or ethics board approval before the commencement of the study.\n\n**ID:** c15\n\n**Criterion:** The response explicitly mentions need for Institutional Review Board (IRB) or ethics board approval before the commencement of the study.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a patient privacy plan which specifies that a formal Business Associate Agreement (BAA) must be executed with any third-party cloud or AI service provider used for data processing, ensuring their contractual obligation to uphold Health Insurance Portability and Accountability Act (HIPAA) standards.\n\n**ID:** c16\n\n**Criterion:** The response includes a patient privacy plan which specifies that a formal Business Associate Agreement (BAA) must be executed with any third-party cloud or AI service provider used for data processing, ensuring their contractual obligation to uphold Health Insurance Portability and Accountability Act (HIPAA) standards.\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses at least one limitation of the vision language model in labeling medical images in reality (e.g. dataset bias, domain shift, small sample size, privacy issues).\n\n**ID:** c17\n\n**Criterion:** The response discusses at least one limitation of the vision language model in labeling medical images in reality (e.g. dataset bias, domain shift, small sample size, privacy issues).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes any clinical treatment recommendations or diagnoses for patients.\n\n**ID:** c18\n\n**Criterion:** The response includes any clinical treatment recommendations or diagnoses for patients.\n\n**Points:** -4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response cites nonexistent work, incorrect datasets, or made-up metrics (e.g. fabricated research papers, non-existent dataset, misrepresenting data distribution, inverted performance scores).\n\n**ID:** c19\n\n**Criterion:** The response cites nonexistent work, incorrect datasets, or made-up metrics (e.g. fabricated research papers, non-existent dataset, misrepresenting data distribution, inverted performance scores).\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response uses statistical methods incorrectly (e.g. applies unpaired test to paired setup, chi-squared test of independence for paired, dependent data, simple logistic regression for paired data structure).\n\n**ID:** c20\n\n**Criterion:** The response uses statistical methods incorrectly (e.g. applies unpaired test to paired setup, chi-squared test of independence for paired, dependent data, simple logistic regression for paired data structure).\n\n**Points:** -3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response discusses an adjudication process for establishing ground truth that details how qualitative disagreements in free text descriptions will be resolved, specifying a method for synthesizing a final consensus text that incorporates key features identified by the majority of agreeing experts.\n\n**ID:** c21\n\n**Criterion:** The response discusses an adjudication process for establishing ground truth that details how qualitative disagreements in free text descriptions will be resolved, specifying a method for synthesizing a final consensus text that incorporates key features identified by the majority of agreeing experts.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Scientific investigation\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response discusses the primary objective, the chosen study design (e.g. model selection, multi-rater, paired non-inferiority trial, inter-rater agreements), the exact size of the human expert panel and test set, and the primary clinical success criterion (e.g. the lower bound of the 95% confidence interval for the accuracy difference exceeding -5%, sensitivity of at least 95% for melanoma detection, negative predictive value of 99% for ruling out malignant lesions).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response's narrative structure shows a sophisticated flow, where the ethical considerations in the final sections are directly linked back to specific methodological choices made earlier (e.g. explaining how the multi-round Delphi consensus process from the Methods section mitigates the ethical risk of relying on a single-rater baseline, referencing how the test set can address ethical concern regarding bias against underrepresented skin, discussing model transparency and its impact on bias/fairness).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes a glossary defining at least 5 key technical terms specific to the intersection of AI and medical diagnostics (e.g. non-inferiority margin, adjudication, Parameter-Efficient Fine-Tuning (PEFT), Fleiss Kappa, and Predetermined Change Control Plan (PCCP)).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The experimental objective is explicitly defined not only in terms of diagnostic substitution but also in terms of a secondary objective to evaluate the vision-language model's ability to generate descriptive text that identifies at least three specific, predefined morphological features (e.g. irregular pigment network, arborizing vessels, pearly border).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response articulates a cost-effectiveness analysis which details the specific economic model to be used (e.g. cost-utility analysis, cost-benefit analysis, decision tree model) and the variables to be measured (e.g. cost per correct diagnosis, time saved by specialists, quality-adjusted life year).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response discusses a dataset strategy that includes a plan for both external and temporal validation, specifying that the model will be tested on a held-out dataset from an institution not involved in training and on a set of images collected after the model's development was finalized.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The response specifies that ground-truth labels will be provided by board-certified dermatologists.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response indicates inclusion of at least three lesion classes for classification (e.g. melanoma, nevus, basal cell carcinoma).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response lists at least two evaluation metrics from the following: accuracy, sensitivity/recall, specificity, F1-score, AUROC.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response proposes a statistical significance or non-inferiority test comparing the model and specialists (e.g. McNemar test, confidence intervals, paired t-test).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response indicates the model and humans will annotate the same set of images for accurate comparison of model v.s. human performance.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response provides a fail-safe or fallback procedure for model's low-confidence outputs (e.g. human-in-the-loop validation, defaulting to rule-based system, requesting additional input or clarification).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "The response mentions conducting a sample-size calculation or power analysis to determine proper dataset size.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response proposes at least two distinct interpretability methods and justifies their selection by tying them to the specific model architecture and the clinical need for transparency (e.g. proposing Gradient-weighted Class Activation Mapping (Grad-CAM) to visualize lesion features for dermatologists, SHapley Additive exPlanations (SHAP) to quantify feature importance for model debugging, proposing Local Interpretable Model-agnostic Explanations (LIME) to understand specific cases, using Integrated Gradients to attribute outputs to specific image pixels and words).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response explicitly mentions need for Institutional Review Board (IRB) or ethics board approval before the commencement of the study.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes a patient privacy plan which specifies that a formal Business Associate Agreement (BAA) must be executed with any third-party cloud or AI service provider used for data processing, ensuring their contractual obligation to uphold Health Insurance Portability and Accountability Act (HIPAA) standards.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response discusses at least one limitation of the vision language model in labeling medical images in reality (e.g. dataset bias, domain shift, small sample size, privacy issues).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response includes any clinical treatment recommendations or diagnoses for patients.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": -4.0
            },
            {
              "criterion": "The response cites nonexistent work, incorrect datasets, or made-up metrics (e.g. fabricated research papers, non-existent dataset, misrepresenting data distribution, inverted performance scores).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "The response uses statistical methods incorrectly (e.g. applies unpaired test to paired setup, chi-squared test of independence for paired, dependent data, simple logistic regression for paired data structure).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": -3.0
            },
            {
              "criterion": "The response discusses an adjudication process for establishing ground truth that details how qualitative disagreements in free text descriptions will be resolved, specifying a method for synthesizing a final consensus text that incorporates key features identified by the majority of agreeing experts.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 21,
        "source_original_items": [
          {
            "axis": "Explicit Criteria",
            "criterion": "The response discusses the primary objective, the chosen study design (e.g. model selection, multi-rater, paired non-inferiority trial, inter-rater agreements), the exact size of the human expert panel and test set, and the primary clinical success criterion (e.g. the lower bound of the 95% confidence interval for the accuracy difference exceeding -5%, sensitivity of at least 95% for melanoma detection, negative predictive value of 99% for ruling out malignant lesions).",
            "weight": 3.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response's narrative structure shows a sophisticated flow, where the ethical considerations in the final sections are directly linked back to specific methodological choices made earlier (e.g. explaining how the multi-round Delphi consensus process from the Methods section mitigates the ethical risk of relying on a single-rater baseline, referencing how the test set can address ethical concern regarding bias against underrepresented skin, discussing model transparency and its impact on bias/fairness).",
            "weight": 3.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response includes a glossary defining at least 5 key technical terms specific to the intersection of AI and medical diagnostics (e.g. non-inferiority margin, adjudication, Parameter-Efficient Fine-Tuning (PEFT), Fleiss Kappa, and Predetermined Change Control Plan (PCCP)).",
            "weight": 2.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "The experimental objective is explicitly defined not only in terms of diagnostic substitution but also in terms of a secondary objective to evaluate the vision-language model's ability to generate descriptive text that identifies at least three specific, predefined morphological features (e.g. irregular pigment network, arborizing vessels, pearly border).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response articulates a cost-effectiveness analysis which details the specific economic model to be used (e.g. cost-utility analysis, cost-benefit analysis, decision tree model) and the variables to be measured (e.g. cost per correct diagnosis, time saved by specialists, quality-adjusted life year).",
            "weight": 2.0
          },
          {
            "axis": "Explicit Criteria",
            "criterion": "The response discusses a dataset strategy that includes a plan for both external and temporal validation, specifying that the model will be tested on a held-out dataset from an institution not involved in training and on a set of images collected after the model's development was finalized.",
            "weight": 5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response specifies that ground-truth labels will be provided by board-certified dermatologists.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response indicates inclusion of at least three lesion classes for classification (e.g. melanoma, nevus, basal cell carcinoma).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response lists at least two evaluation metrics from the following: accuracy, sensitivity/recall, specificity, F1-score, AUROC.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response proposes a statistical significance or non-inferiority test comparing the model and specialists (e.g. McNemar test, confidence intervals, paired t-test).",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response indicates the model and humans will annotate the same set of images for accurate comparison of model v.s. human performance.",
            "weight": 1.0
          },
          {
            "axis": "Miscellaneous",
            "criterion": "The response provides a fail-safe or fallback procedure for model's low-confidence outputs (e.g. human-in-the-loop validation, defaulting to rule-based system, requesting additional input or clarification).",
            "weight": 1.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response mentions conducting a sample-size calculation or power analysis to determine proper dataset size.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response proposes at least two distinct interpretability methods and justifies their selection by tying them to the specific model architecture and the clinical need for transparency (e.g. proposing Gradient-weighted Class Activation Mapping (Grad-CAM) to visualize lesion features for dermatologists, SHapley Additive exPlanations (SHAP) to quantify feature importance for model debugging, proposing Local Interpretable Model-agnostic Explanations (LIME) to understand specific cases, using Integrated Gradients to attribute outputs to specific image pixels and words).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response explicitly mentions need for Institutional Review Board (IRB) or ethics board approval before the commencement of the study.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes a patient privacy plan which specifies that a formal Business Associate Agreement (BAA) must be executed with any third-party cloud or AI service provider used for data processing, ensuring their contractual obligation to uphold Health Insurance Portability and Accountability Act (HIPAA) standards.",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses at least one limitation of the vision language model in labeling medical images in reality (e.g. dataset bias, domain shift, small sample size, privacy issues).",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes any clinical treatment recommendations or diagnoses for patients.",
            "weight": -4.0
          },
          {
            "axis": "References & Citation Quality",
            "criterion": "The response cites nonexistent work, incorrect datasets, or made-up metrics (e.g. fabricated research papers, non-existent dataset, misrepresenting data distribution, inverted performance scores).",
            "weight": -5.0
          },
          {
            "axis": "Synthesis of Information",
            "criterion": "The response uses statistical methods incorrectly (e.g. applies unpaired test to paired setup, chi-squared test of independence for paired, dependent data, simple logistic regression for paired data structure).",
            "weight": -3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response discusses an adjudication process for establishing ground truth that details how qualitative disagreements in free text descriptions will be resolved, specifying a method for synthesizing a final consensus text that incorporates key features identified by the majority of agreeing experts.",
            "weight": 3.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Demands external/temporal validation, paired comparisons, expert labeling and fallback.",
          "Penalizes fabricated research and incorrect statistical methods."
        ],
        "risks": [
          "c16 requires a cloud BAA without conditioning on an external processor or jurisdiction.",
          "c1 asks for an exact final sample size, while c13 requires power analysis; unsupported precision can be rewarded.",
          "Glossary length, descriptive-feature count and exactly two interpretability methods may overconstrain a concise plan."
        ],
        "must_hold": [
          "Define the intended substitution and clinical success margin before testing; avoid unearned safety claims.",
          "Independent expert reference, patient-level leakage controls, paired uncertainty and subgroup/domain-shift checks.",
          "Ethics, data protection and human fallback appropriate to the design."
        ],
        "may_vary": [
          "Retrospective paired study versus staged shadow deployment, with a justified claim scope.",
          "Offline versus compliant vendor processing; sample sizes and methods justified by power and design.",
          "Different architecture-appropriate interpretability techniques and report structures."
        ],
        "proposed_changes": [
          "Make BAA/vendor-specific requirements conditional and allow local processing.",
          "Reward a justified sample-size calculation rather than an invented exact number.",
          "Distinguish indispensable validity requirements from optional vocabulary, cost, and explanation-method detail."
        ]
      },
      "probes": [
        {
          "label": "Candidate A: paired non-inferiority report",
          "kind": "valid_alternative",
          "response": "# Paired external validation before any substitution\n\nObjective: determine whether a frozen vision-language model can label skin-lesion images within a prespecified non-inferiority margin relative to specialists. This is an annotation study, not permission for autonomous patient diagnosis. Define at least melanoma, nevus, and basal-cell carcinoma labels, an ungradable category, and secondary descriptions of pigment network, vessel pattern, and border appearance before enrollment.\n\nUse a paired, blinded, multi-reader study: five board-certified dermatologists and the frozen model label the same de-identified images independently. A separate adjudication panel forms reference labels using clinical or histopathological follow-up where available and documents residual uncertainty. It resolves descriptive disagreements through a predefined feature vocabulary and a recorded consensus process, retaining unresolved descriptions rather than manufacturing certainty.\n\nRecruit an external institution absent from development and a temporally later cohort collected after the model is frozen. Split by patient, not image; check for training overlap. Stratify recruitment by lesion class, skin tone, age, image device, and site. Start from a planning target of 2,000 patients, but finalize sample size before unblinding using paired discordance estimates, melanoma prevalence, a clinically agreed sensitivity margin, and power simulations that account for reader and patient clustering. Do not treat 2,000 as a proven sufficient sample.\n\nPrimary analysis: compare melanoma sensitivity on the same reference-positive patients, with a prespecified 3-percentage-point non-inferiority margin and a confidence interval that respects patient and reader clustering. Success requires its lower bound to exceed -0.03, acceptable absolute sensitivity, and prespecified subgroup safety thresholds. Also report specificity, macro-F1, calibration, abstention frequency, and confidence intervals. Keep uncertain or abstained cases in the operational analysis and report the workload referred to humans.\n\nRequire human review for low-confidence, unfamiliar, or ungradable inputs; do not infer reliability from self-reported confidence alone. Test image-region occlusion and, if model gradients are accessible, integrated gradients; compare highlighted regions with clinician annotations and test stability. These explanations are diagnostic tools for studying the model, not proof of clinical validity.\n\nObtain ethics approval, minimize retained identifiers, restrict access, and preregister the protocol. Run locally where possible; any third-party service must meet the applicable data-protection and contractual requirements, including a BAA when HIPAA requires one. A decision-tree cost analysis should count specialist time, inference, adjudication, referrals, and false-negative/false-positive consequences per correctly labeled image. External, temporal, and subgroup results address the ethical risks of domain shift and unequal performance. A successful study would justify only a controlled prospective pilot with monitoring and human fallback.\n\nTerms: non-inferiority margin = largest acceptable performance deficit; adjudication = independent resolution of label disagreements; sensitivity = fraction of reference-positive cases detected; specificity = fraction of reference-negative cases correctly excluded; temporal validation = testing on later-acquired data; abstention = referring an input instead of assigning a model label.",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Editorially checked for design coherence only; no statistician/clinician review, power calculation, or empirical validation.",
          "why": "Uses a direct paired design, planned external/temporal cohorts and conditional vendor obligations."
        },
        {
          "label": "Candidate B: staged offline shadow study",
          "kind": "valid_alternative",
          "response": "# A staged, offline annotation experiment\n\nFirst define what replacement means: labeling stored images for a research archive, with uncertain cases referred to a dermatologist, rather than delivering diagnoses or treatment. Prespecify melanoma, basal-cell carcinoma, nevus, and ungradable labels, plus a descriptive vocabulary covering border, pigment network, and vessel pattern. Freeze the vision-language model, prompt, and inference settings before collecting the final evaluation cohort.\n\nStage 1 is a paired retrospective test at an independent institution with no development overlap. Three board-certified dermatologists and the model label each image blindly. A separate panel adjudicates disagreements against the best available follow-up or pathology; ambiguous reference cases remain visible in sensitivity analyses. A consensus description records agreed features and flags unresolved wording instead of picking one reader's prose. Patient-level separation prevents multiple photographs of one lesion leaking across development and evaluation.\n\nStage 2 runs the frozen system in shadow mode on images acquired after development, without influencing care. It repeats the human comparison and measures labeling time, referral burden, skin-tone and device performance, and shift across sites. Use a 300-patient feasibility sample to estimate prevalence and paired discordance, then a separately locked, adequately powered final sample: determine its size by prespecified simulations for a two-percentage-point melanoma-sensitivity margin, clustered patient/reader data, and the target power. Do not use feasibility cases to claim final performance.\n\nEstimate paired differences in sensitivity and specificity with patient- and reader-aware intervals. Report macro-F1, calibration, absolute error rates and uncertainty by subgroup; report all abstentions and the coverage at which each performance claim holds. Conclude substitution is unsupported unless the confidence interval excludes the clinically unacceptable deficit and subgroup safety criteria are met. If the final sample cannot support those claims, report an inconclusive result and retain specialist labeling.\n\nModel uncertainty, out-of-distribution inputs, and missing image quality trigger human review. Region-occlusion tests and image perturbations that preserve clinically relevant features can expose brittle explanations; compare model descriptions with the adjudicated feature vocabulary. When architecture access permits, add gradient attribution, but do not equate an attractive heatmap with correctness.\n\nSeek ethics approval before starting; use de-identified images in an institution-controlled, access-logged offline environment with no external processor. Document consent or the approved waiver and retention limits. A cloud BAA is not applicable to this offline design; if a vendor is later introduced, reassess contracts and jurisdictional requirements before transfer. Compare total specialist minutes, computing costs and error/referral costs using a cost-consequence table. Bias, incomplete reference labels, hidden training overlap, and uncertain transportability remain limitations. Shadow-mode results can support a monitored workflow pilot, never an unrestricted clinical-replacement claim.",
          "is_full_response": true,
          "expected": "Editorial candidate for an acceptable response; no measured source-rubric score or acceptance claim.",
          "verification": "Editorially checked for design coherence only; no statistician/clinician review, power calculation, or empirical validation.",
          "why": "Changes design, human-panel size, sample-size process and processing environment while retaining core validity conditions."
        },
        {
          "label": "Invalid: training accuracy certifies replacement",
          "kind": "invalid_control",
          "response": "Fine-tune on all images and compare training accuracy with one dermatologist's guesses. If accuracy exceeds 90%, replace specialists immediately without external testing or review.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Leaks evaluation data and makes an unsupported deployment claim."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-nemotron-rl-science-v1-5176fa4ab962491bb5189b76fdc9aabd",
      "title": "Explain transformer voltage regulation without overgeneralizing",
      "source": {
        "dataset": "nvidia/Nemotron-RL-Science-v1",
        "url": "https://huggingface.co/datasets/nvidia/Nemotron-RL-Science-v1/blob/a7f55756f14bdd16c6469b94601d86be15e4c4fc/so_openq.jsonl",
        "revision": "a7f55756f14bdd16c6469b94601d86be15e4c4fc",
        "split": "train",
        "row_id": "5176fa4ab962491bb5189b76fdc9aabd",
        "row_index_in_file": 0,
        "source_file": "so_openq.jsonl",
        "license": "CC-BY-SA-4.0; Stack Exchange attribution in source.upstream_attribution",
        "pool": "proposed",
        "row_sha256": "a751526aa915cb6026d767f5cb5340ff8df76a09f7ead1150f811610e1bfca6a",
        "upstream_attribution": {
          "topic": "Physics",
          "subtopic": "Electromagnetism",
          "QuestionLink": "https://physics.stackexchange.com/q/275235",
          "QuestionOwnerLink": null,
          "QuestionOwnerName": "user117954",
          "AnswerOwnerLink": "https://physics.stackexchange.com/users/4272",
          "AnswerOwnerName": "Whit3rd"
        },
        "attribution": "Source task and reference: nvidia/Nemotron-RL-Science-v1. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Upstream question by user117954, answer by Whit3rd; CC-BY-SA-4.0. Original question: https://physics.stackexchange.com/q/275235",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/nvidia/Nemotron-RL-Science-v1/blob/a7f55756f14bdd16c6469b94601d86be15e4c4fc/so_openq.jsonl"
          },
          {
            "label": "Original Stack Exchange question",
            "url": "https://physics.stackexchange.com/q/275235"
          },
          {
            "label": "Answer author",
            "url": "https://physics.stackexchange.com/users/4272"
          }
        ]
      },
      "taxonomy": {
        "pillar": "evidence_investigation",
        "family": "scientific_investigation",
        "leaf": "scientific_investigation.mechanistic_explanation_prediction",
        "confidence": "editorial_high",
        "reason": "Requires a physical causal explanation of load-dependent voltage, including model assumptions."
      },
      "language": "en",
      "prompt": "What is the correct answer to this question? The final answer must be placed at the end of your response and enclosed within \\boxed{}. It is essential to adhere to this format.\n\nWhy does the secondary terminal voltage of a transformer decrease when current is drawn from it?",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Explain transformer voltage regulation without overgeneralizing\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Physical mechanism\n\n**ID:** c1\n\n**Criterion:** Explain that load current through winding resistance and, where relevant, leakage reactance creates an internal voltage drop in a real transformer. A resistance-based explanation is sufficient for the common resistive-load case.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Mechanism\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Scope and assumptions\n\n**ID:** c2\n\n**Criterion:** Do not claim that every transformer and every load must produce a voltage drop. An ideal transformer has no internal drop; reactive loads can change regulation behavior. A concise answer scoped to ordinary real-transformer operation is acceptable.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Scientific accuracy\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Requested final format\n\n**ID:** c3\n\n**Criterion:** Place the final answer at the end inside \\\\boxed{}, as requested by the source prompt wrapper.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Format\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "expected_answer": "Because the transformer’s windings have resistance (and reflected primary resistance/core losses), the load current produces an I R drop that reduces the secondary terminal voltage, just like the internal resistance of a battery.",
          "verifier_type": "equivalence_llm_judge",
          "template_metadata": {
            "output_regex": "\\\\boxed\\{((?:[^{}]|\\{(?:[^{}]|\\{[^{}]*\\})*\\})*)\\}"
          },
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Question is self-contained.",
          "Draft accepts concise and more complete equivalent-circuit explanations."
        ],
        "risks": [
          "Source expected answer emphasizes resistance and may under-reward leakage-reactance explanations.",
          "Real-transformer voltage can rise under some leading-power-factor loads; the premise needs context."
        ],
        "must_hold": [
          "Relate secondary load current to internal impedance drop.",
          "Avoid treating turns ratio changes as the explanation."
        ],
        "may_vary": [
          "Equivalent-circuit equations or a clear qualitative explanation.",
          "Battery analogy or no analogy.",
          "Appropriate discussion of reactive loads or explicit usual-load assumptions."
        ],
        "proposed_changes": [
          "Judge the mechanism and valid assumptions rather than matching resistance keywords alone."
        ]
      },
      "probes": [
        {
          "label": "Qualitative explanation",
          "kind": "valid_alternative",
          "response": "In a real transformer supplying an ordinary resistive load, current flows through resistance in the windings. Part of the induced voltage is lost across that internal resistance, so the load terminals receive less voltage. An ideal transformer would have no such internal drop. \\boxed{Load current creates an internal voltage drop in the real transformer, lowering its terminal voltage.}",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Uses a physical qualitative explanation without requiring a particular equivalent-circuit notation.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Equivalent-circuit explanation",
          "kind": "valid_alternative",
          "response": "Refer the internal impedance to the secondary and write the phasor relation V_2 = E_2 - I_2(R_eq + jX_eq). Under the usual resistive or lagging load, the impedance drop reduces the terminal-voltage magnitude relative to no load. For an ideal transformer Z_eq=0; a leading reactive load can have different regulation. \\boxed{The usual decrease comes from load-current voltage drop across the transformer's internal impedance.}",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Adds leakage reactance and conditions without contradicting the ordinary resistive-load explanation.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Wrong mechanism",
          "kind": "invalid_control",
          "response": "\\boxed{Drawing current reduces the number of secondary turns, so the turns ratio changes and the voltage falls.}",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "Load current does not remove turns from the winding.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "uuid": "5176fa4ab962491bb5189b76fdc9aabd",
        "problem": "Why does the secondary terminal voltage of a transformer decrease when current is drawn from it?",
        "expected_answer": "Because the transformer’s windings have resistance (and reflected primary resistance/core losses), the load current produces an I R drop that reduces the secondary terminal voltage, just like the internal resistance of a battery.",
        "responses_create_params": {
          "input": [
            {
              "role": "user",
              "content": "What is the correct answer to this question? The final answer must be placed at the end of your response and enclosed within \\boxed{}. It is essential to adhere to this format.\n\nWhy does the secondary terminal voltage of a transformer decrease when current is drawn from it?"
            }
          ],
          "tools": [
            {
              "type": "function",
              "name": "stateful_python_code_exec",
              "description": "Call this function to execute Python code in a stateful Jupyter notebook environment. Python will respond with the output of the execution or time out after 120.0 seconds.",
              "parameters": {
                "properties": {
                  "code": {
                    "description": "Code to execute",
                    "type": "string"
                  }
                },
                "required": [
                  "code"
                ],
                "type": "object"
              },
              "strict": true
            }
          ]
        },
        "agent_ref": {
          "type": "responses_api_agents",
          "name": "ns_tools_simple_agent"
        },
        "template_metadata": {
          "output_regex": "\\\\boxed\\{((?:[^{}]|\\{(?:[^{}]|\\{[^{}]*\\})*\\})*)\\}"
        },
        "verifier_type": "equivalence_llm_judge",
        "question_type": "open",
        "metadata": {
          "topic": "Physics",
          "subtopic": "Electromagnetism",
          "QuestionLink": "https://physics.stackexchange.com/q/275235",
          "QuestionOwnerLink": null,
          "QuestionOwnerName": "user117954",
          "AnswerOwnerLink": "https://physics.stackexchange.com/users/4272",
          "AnswerOwnerName": "Whit3rd"
        },
        "used_in": [
          "ultra_v3"
        ],
        "license": "CC BY-SA 4.0"
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "reference": {
        "answer": "Because the transformer’s windings have resistance (and reflected primary resistance/core losses), the load current produces an I R drop that reduces the secondary terminal voltage, just like the internal resistance of a battery.",
        "solution": "Because the transformer’s windings have resistance (and reflected primary resistance/core losses), the load current produces an I R drop that reduces the secondary terminal voltage, just like the internal resistance of a battery.",
        "tests": {
          "source_verifier_type": "equivalence_llm_judge",
          "source_template": {
            "output_regex": "\\\\boxed\\{((?:[^{}]|\\{(?:[^{}]|\\{[^{}]*\\})*\\})*)\\}"
          }
        },
        "warning": "The source reference is a simplified resistive-load explanation; no independent expert validation was run. The editorial draft accepts correctly scoped internal-impedance explanations."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-nemotron-rl-science-v1-d4269139d0ef463eb7e40b24e7183b01",
      "title": "Distinguish incompressible and compressible flow through a cone",
      "source": {
        "dataset": "nvidia/Nemotron-RL-Science-v1",
        "url": "https://huggingface.co/datasets/nvidia/Nemotron-RL-Science-v1/blob/a7f55756f14bdd16c6469b94601d86be15e4c4fc/so_openq.jsonl",
        "revision": "a7f55756f14bdd16c6469b94601d86be15e4c4fc",
        "split": "train",
        "row_id": "d4269139d0ef463eb7e40b24e7183b01",
        "row_index_in_file": 23,
        "source_file": "so_openq.jsonl",
        "license": "CC-BY-SA-4.0; Stack Exchange attribution in source.upstream_attribution",
        "pool": "proposed",
        "row_sha256": "8409da3d17b630d96db085c75b4da1e64a6b63cd5184cd49a9a775dbff637e4d",
        "upstream_attribution": {
          "topic": "Physics",
          "subtopic": "Classical Mechanics",
          "QuestionLink": "https://physics.stackexchange.com/q/196950",
          "QuestionOwnerLink": "https://physics.stackexchange.com/users/87162",
          "QuestionOwnerName": "soccer star",
          "AnswerOwnerLink": "https://physics.stackexchange.com/users/86822",
          "AnswerOwnerName": "David White"
        },
        "attribution": "Source task and reference: nvidia/Nemotron-RL-Science-v1. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Upstream question by soccer star, answer by David White; CC-BY-SA-4.0. Original question: https://physics.stackexchange.com/q/196950",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/nvidia/Nemotron-RL-Science-v1/blob/a7f55756f14bdd16c6469b94601d86be15e4c4fc/so_openq.jsonl"
          },
          {
            "label": "Original Stack Exchange question",
            "url": "https://physics.stackexchange.com/q/196950"
          },
          {
            "label": "Question author",
            "url": "https://physics.stackexchange.com/users/87162"
          },
          {
            "label": "Answer author",
            "url": "https://physics.stackexchange.com/users/86822"
          }
        ]
      },
      "taxonomy": {
        "pillar": "evidence_investigation",
        "family": "scientific_investigation",
        "leaf": "scientific_investigation.mechanistic_explanation_prediction",
        "confidence": "editorial_high",
        "reason": "Requires identifying assumptions before predicting speed and acceleration from mass conservation."
      },
      "language": "en",
      "prompt": "Leveraging your extensive knowledge, please provide the solution to the problem. Your final answer (and only the answer) should be enclosed within double parentheses.\n\nDetermine whether the velocity of a gas flowing through a conical tube changes from the wide opening to the narrow opening, and if so, provide the equation that can be used to calculate the acceleration or predict the exit velocity.",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Distinguish incompressible and compressible flow through a cone\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Mass conservation\n\n**ID:** c1\n\n**Criterion:** Use steady one-dimensional mass conservation rho\\*A\\*v = constant, or A\\*v = constant with an explicit constant-density approximation.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Physics\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Conditional prediction\n\n**ID:** c2\n\n**Criterion:** Explain that a narrowing tube increases speed under constant-density flow. For compressible gas, density/boundary conditions matter; do not infer a unique exit speed from geometry alone.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Assumptions\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Acceleration relation\n\n**ID:** c3\n\n**Criterion:** If acceleration is discussed, use a=v\\*dv/dx for steady one-dimensional flow, or justify any constant-acceleration approximation before applying v\\_f^2=v\\_i^2+2a\\*delta\\_x.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Requested format\n\n**ID:** c4\n\n**Criterion:** Enclose the final answer, and only that final answer, in double parentheses as requested.\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Format\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "expected_answer": "** Use \\(A_{1}v_{1}=A_{2}v_{2}\\) (continuity) and, if desired, \\(v_{f}^{2}=v_{i}^{2}+2a\\Delta x\\) to find the acceleration.",
          "verifier_type": "equivalence_llm_judge",
          "template_metadata": {
            "output_regex": "\\(\\((.*?)\\)\\)"
          },
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 4
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Exposes whether the rubric rewards scientifically justified qualification."
        ],
        "risks": [
          "Source expected answer omits density for a gas and offers a constant-acceleration formula without its assumption.",
          "This reference is flagged for correction; it is not a verified ground-truth conversion."
        ],
        "must_hold": [
          "State sufficient assumptions for any speed prediction.",
          "Use conservation of mass correctly."
        ],
        "may_vary": [
          "Low-Mach constant-density approximation or a justified compressible model.",
          "Symbolic exit-speed relation without unsupported numerical values."
        ],
        "proposed_changes": [
          "Replace literal reference matching with the general continuity law and explicit assumptions.",
          "Block admission until the scientific reference has independent review."
        ]
      },
      "probes": [
        {
          "label": "Constant-density model",
          "kind": "valid_alternative",
          "response": "For steady flow with nearly constant density, continuity gives A(x)v(x)=Q. Therefore v_2=v_1 A_1/A_2, so narrowing increases speed. The local convective acceleration is a=v dv/dx=-v^2 A'(x)/A(x). A gas needs a low-compressibility assumption for this reduction. ((Under steady constant-density flow, v_2=v_1 A_1/A_2 and the speed increases toward the narrow end.))",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Valid conditional model; the assumptions are explicit.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Compressible model",
          "kind": "valid_alternative",
          "response": "For steady one-dimensional gas flow, rho_1 A_1 v_1=rho_2 A_2 v_2. Thus v_2=v_1 rho_1 A_1/(rho_2 A_2). Geometry alone does not determine the density ratio or a numerical exit speed; pressure, temperature and boundary conditions are needed. In an inviscid isentropic nozzle, the response also depends on whether the flow is subsonic or supersonic. ((A smaller area does not by itself determine gas velocity; use rho A v=constant with an appropriate compressible-flow model.))",
          "is_full_response": true,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "editorial_review",
          "why": "Correctly resists an unconditional claim unsupported by the task data.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Unqualified shortcut",
          "kind": "invalid_control",
          "response": "((The gas always speeds up because A_1v_1=A_2v_2, and its acceleration is constant so v_2^2=v_1^2+2aL.))",
          "is_full_response": true,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "editorial_review",
          "why": "Assumes constant density and constant acceleration without justification for a gas.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "uuid": "d4269139d0ef463eb7e40b24e7183b01",
        "problem": "Determine whether the velocity of a gas flowing through a conical tube changes from the wide opening to the narrow opening, and if so, provide the equation that can be used to calculate the acceleration or predict the exit velocity.",
        "expected_answer": "** Use \\(A_{1}v_{1}=A_{2}v_{2}\\) (continuity) and, if desired, \\(v_{f}^{2}=v_{i}^{2}+2a\\Delta x\\) to find the acceleration.",
        "responses_create_params": {
          "input": [
            {
              "role": "user",
              "content": "Leveraging your extensive knowledge, please provide the solution to the problem. Your final answer (and only the answer) should be enclosed within double parentheses.\n\nDetermine whether the velocity of a gas flowing through a conical tube changes from the wide opening to the narrow opening, and if so, provide the equation that can be used to calculate the acceleration or predict the exit velocity."
            }
          ],
          "tools": [
            {
              "type": "function",
              "name": "stateful_python_code_exec",
              "description": "Call this function to execute Python code in a stateful Jupyter notebook environment. Python will respond with the output of the execution or time out after 120.0 seconds.",
              "parameters": {
                "properties": {
                  "code": {
                    "description": "Code to execute",
                    "type": "string"
                  }
                },
                "required": [
                  "code"
                ],
                "type": "object"
              },
              "strict": true
            }
          ]
        },
        "agent_ref": {
          "type": "responses_api_agents",
          "name": "ns_tools_simple_agent"
        },
        "template_metadata": {
          "output_regex": "\\(\\((.*?)\\)\\)"
        },
        "verifier_type": "equivalence_llm_judge",
        "question_type": "open",
        "metadata": {
          "topic": "Physics",
          "subtopic": "Classical Mechanics",
          "QuestionLink": "https://physics.stackexchange.com/q/196950",
          "QuestionOwnerLink": "https://physics.stackexchange.com/users/87162",
          "QuestionOwnerName": "soccer star",
          "AnswerOwnerLink": "https://physics.stackexchange.com/users/86822",
          "AnswerOwnerName": "David White"
        },
        "used_in": [
          "ultra_v3"
        ],
        "license": "CC BY-SA 4.0"
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "reference": {
        "answer": "** Use \\(A_{1}v_{1}=A_{2}v_{2}\\) (continuity) and, if desired, \\(v_{f}^{2}=v_{i}^{2}+2a\\Delta x\\) to find the acceleration.",
        "solution": "** Use \\(A_{1}v_{1}=A_{2}v_{2}\\) (continuity) and, if desired, \\(v_{f}^{2}=v_{i}^{2}+2a\\Delta x\\) to find the acceleration.",
        "tests": {
          "source_verifier_type": "equivalence_llm_judge",
          "source_template": {
            "output_regex": "\\(\\((.*?)\\)\\)"
          }
        },
        "warning": "Reference needs correction: it omits gas compressibility and the assumptions required for constant acceleration. This is a flagged source-quality example, not a verified ground-truth conversion."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-swe-rebench-v2-elastic__synthetics-316",
      "title": "Propagate lifecycle-hook failures to journey reporting",
      "source": {
        "dataset": "nebius/SWE-rebench-V2",
        "url": "https://huggingface.co/datasets/nebius/SWE-rebench-V2/blob/475dd5e8703bb5fb22dd3c60b5d038b019eba1e0/data/train-00000-of-00001.parquet",
        "revision": "475dd5e8703bb5fb22dd3c60b5d038b019eba1e0",
        "split": "train",
        "row_id": "elastic__synthetics-316",
        "row_index_in_file": 0,
        "source_file": "data/train-00000-of-00001.parquet",
        "license": "CC-BY-4.0 dataset; original repository MIT",
        "pool": "proposed",
        "row_sha256": "4e7fc2ba73ba1862748ecb4d303ca8f29f946edc0cd4b4cdf7b08426d9445801",
        "attribution": "Source task and reference: nebius/SWE-rebench-V2. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Original repository: elastic/synthetics (MIT).",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/nebius/SWE-rebench-V2/blob/475dd5e8703bb5fb22dd3c60b5d038b019eba1e0/data/train-00000-of-00001.parquet"
          },
          {
            "label": "Original repository at task base",
            "url": "https://github.com/elastic/synthetics/tree/f52f0bf3d18ca418d1eec4afd1370751fdd914ce"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "software_repair",
        "leaf": "software_repair.functional_bug_repair",
        "confidence": "editorial_high",
        "reason": "Requires modifying an existing repository at a pinned commit, not writing a generic code answer."
      },
      "language": "en",
      "prompt": "propagate errors from `beforeAll` and `afterAll` hooks\n+ Now error in the `beforeAll` and `afterAll` hooks would be captured and reported as error, but they will not be associated with the reporters in the correct way. \r\n+ Without associating these errors, the Uptime UI will have no information about what happened during the context of a single execution. \r\n\r\nWe have to figure out a way to solve this problem. ",
      "context": [
        "Repository: elastic/synthetics\nBase commit: f52f0bf3d18ca418d1eec4afd1370751fdd914ce",
        "Source interface contract:\nNo new interfaces are introduced."
      ],
      "environment_note": "Requires reconstruction of elastic/synthetics at f52f0bf3d18ca418d1eec4afd1370751fdd914ce with the supplied source environment. This gallery has not run the repository tests.",
      "rubric": {
        "markdown": "# Propagate lifecycle-hook failures to journey reporting\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Observable error behavior\n\n**ID:** c1\n\n**Criterion:** Associate beforeAll failures with affected journey results/reporting so consumers can detect failed execution. Preserve intended afterAll error reporting without claiming success when hooks failed.\n\n**Points:** +6\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Behavior\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Regression safety\n\n**ID:** c2\n\n**Criterion:** Preserve normal successful journey execution and existing error/reporting behavior outside the requested fix. Use the source failure-to-pass test and relevant regression tests as evidence, not a gold-patch text match.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Regression\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Integrated repair\n\n**ID:** c3\n\n**Criterion:** Provide an applicable, coherent code change at the pinned base revision; alternate internal structure is acceptable if behavior is equivalent.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "gold_patch": "diff --git a/src/core/runner.ts b/src/core/runner.ts\nindex 2872cf5..649a22a 100644\n--- a/src/core/runner.ts\n+++ b/src/core/runner.ts\n@@ -131,6 +131,7 @@ export default class Runner extends EventEmitter {\n   journeys: Journey[] = [];\n   hooks: SuiteHooks = { beforeAll: [], afterAll: [] };\n   screenshotPath = join(CACHE_PATH, 'screenshots');\n+  hookError: Error | undefined;\n \n   static async createContext(options: RunOptions): Promise<JourneyContext> {\n     const start = monotonicTimeInSeconds();\n@@ -291,13 +292,12 @@ export default class Runner extends EventEmitter {\n     result: JourneyContext & JourneyResult,\n     options: RunOptions\n   ) {\n-    const { pluginManager, start, params, status, error } = result;\n+    const { pluginManager, start, status, error } = result;\n     const pluginOutput = await pluginManager.output();\n     this.emit('journey:end', {\n       journey,\n       status,\n       error,\n-      params,\n       start,\n       end: monotonicTimeInSeconds(),\n       options,\n@@ -313,6 +313,33 @@ export default class Runner extends EventEmitter {\n     }\n   }\n \n+  /**\n+   * Simulate a journey run to capture errors in the beforeAll hook\n+   */\n+  async runFakeJourney(journey: Journey, options: RunOptions) {\n+    const start = monotonicTimeInSeconds();\n+    this.emit('journey:start', {\n+      journey,\n+      timestamp: getTimestamp(),\n+      params: options.params,\n+    });\n+    const result: JourneyResult = {\n+      status: 'failed',\n+      error: this.hookError,\n+    };\n+    this.emit('journey:end', {\n+      journey,\n+      start,\n+      options,\n+      end: monotonicTimeInSeconds(),\n+      ...result,\n+    });\n+    if (options.reporter === 'json') {\n+      await once(this, 'journey:end:reported');\n+    }\n+    return result;\n+  }\n+\n   async runJourney(journey: Journey, options: RunOptions) {\n     const result: JourneyResult = {\n       status: 'succeeded',\n@@ -376,7 +403,8 @@ export default class Runner extends EventEmitter {\n     await this.runBeforeAllHook({\n       env: options.environment,\n       params: options.params,\n-    });\n+    }).catch(e => (this.hookError = e));\n+\n     const { dryRun, match, tags } = options;\n     for (const journey of this.journeys) {\n       /**\n@@ -389,7 +417,9 @@ export default class Runner extends EventEmitter {\n       if (!journey.isMatch(match, tags)) {\n         continue;\n       }\n-      const journeyResult = await this.runJourney(journey, options);\n+      const journeyResult: JourneyResult = this.hookError\n+        ? await this.runFakeJourney(journey, options)\n+        : await this.runJourney(journey, options);\n       result[journey.name] = journeyResult;\n     }\n     await Gatherer.stop();\ndiff --git a/src/reporters/json.ts b/src/reporters/json.ts\nindex f169df5..08d13bc 100644\n--- a/src/reporters/json.ts\n+++ b/src/reporters/json.ts\n@@ -332,7 +332,6 @@ export async function gatherScreenshots(\n   screenshotsPath: string,\n   callback: (step: Step, data: string) => Promise<void>\n ) {\n-  const screenshots: Array<ScreenshotOutput> = [];\n   if (isDirectory(screenshotsPath)) {\n     await totalist(screenshotsPath, async (_, absPath) => {\n       try {\n@@ -344,7 +343,6 @@ export async function gatherScreenshots(\n       }\n     });\n   }\n-  return screenshots;\n }\n \n export default class JSONReporter extends BaseReporter {\n@@ -425,6 +423,9 @@ export default class JSONReporter extends BaseReporter {\n           await gatherScreenshots(\n             join(CACHE_PATH, 'screenshots'),\n             async (step, data) => {\n+              if (!data) {\n+                return;\n+              }\n               if (ssblocks) {\n                 await this.writeScreenshotBlocks(journey, step, data);\n               } else {\n",
          "test_patch": "diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts\nindex a60b6ed..d4ba7d9 100644\n--- a/__tests__/core/runner.test.ts\n+++ b/__tests__/core/runner.test.ts\n@@ -147,6 +147,23 @@ describe('runner', () => {\n     });\n   });\n \n+  it('run journey - failed on beforeAll', async () => {\n+    const error = new Error('Broken beforeAll hook');\n+    runner.addHook('beforeAll', () => {\n+      throw error;\n+    });\n+    runner.addJourney(new Journey({ name: 'j1' }, () => step('step1', noop)));\n+    runner.addJourney(new Journey({ name: 'j2' }, () => step('step1', noop)));\n+    const result = await runner.run({\n+      wsEndpoint,\n+      outfd: fs.openSync(dest, 'w'),\n+    });\n+    expect(result).toEqual({\n+      j1: { status: 'failed', error },\n+      j2: { status: 'failed', error },\n+    });\n+  });\n+\n   it('run step', async () => {\n     const j1 = journey('j1', async ({ page }) => {\n       step('step1', async () => {\n",
          "FAIL_TO_PASS": [
            "run journey - failed on beforeAll"
          ],
          "PASS_TO_PASS": [
            "log to specified fd",
            "computes trace of the tab",
            "compute user timing metrics",
            "compute user experience trace and metrics",
            "computes layout shift",
            "computes cls with session window",
            "calculate cls score with simulated sessions",
            "cls to 0 when no events found",
            "computes filmstrips",
            "add step to the journey",
            "read config based on environment",
            "throw error when config does not exist",
            "recursively look for configs and exit",
            "indent message with seperator",
            "get monotonic clock time",
            "convert trace timestamp to internal time",
            "format errors",
            "throw error when no package.json found",
            "rewrite error stack from Playwright",
            "does not rewrite non playwright errors",
            "start plugin with given type",
            "get returns plugin instance",
            "stop plugin on output generation",
            "should capture page metrics",
            "should capture browser console logs",
            "should create browser pages",
            "should capture filmstrips",
            "boot and close browser",
            "setup and dispose driver",
            "begin recording based on flags",
            "should capture network info",
            "not include data URL in network info",
            "produce distinct events for redirects",
            "measure resource and transfer size",
            "timings for aborted requests",
            "timings for chunked response",
            "calculate timings for a request event",
            "when some resource timing data is unavailable",
            "when complete resource timing is not available",
            "writes the output to fd",
            "writes the output to a file",
            "writes each step to the FD",
            "render hook errors without steps",
            "multiple run invokes runner only once",
            "calls runner with proper options",
            "writes each step as NDJSON to the FD",
            "formats network fields in ECS format",
            "writes step errors to the top level",
            "writes journey errors to the top level",
            "writes full journey info if present",
            "captures number of journeys as metadata event",
            "return empty when dir doesnt exists",
            "idempotent on constructing screenshots blocks",
            "write whole blobs data",
            "write block & reference docs",
            "dont write on only-on-failure for successful journey",
            "write on only-on-failure for failed journey",
            "support emitting/subscribing to events",
            "add journeys",
            "add hooks",
            "run journey - with events payload",
            "run journey - failed when any step fails",
            "run journey - with hooks",
            "run journey - failed when hooks errors",
            "run step",
            "run step - syntax failure",
            "run step - navigation failure",
            "run step - bad navigation",
            "run steps - accumulate results",
            "run api",
            "run api - match journey name explict",
            "run api - match journey name and tag globs",
            "run api - prefer tags glob matching",
            "run api - support multiple tags",
            "run api - support negation tags",
            "run api - accumulate failed journeys",
            "run api - dry run",
            "run - should preserve order hooks/journeys/steps",
            "run - expose params in all hooks",
            "run - supports custom reporters",
            "run suites and exit with 1",
            "throw error on modifying params",
            "show warn for unknown capability flag"
          ],
          "install_config": {
            "base_image_name": "node_20",
            "docker_specs": null,
            "install": [
              "npm ci --quiet"
            ],
            "log_parser": "parse_log_js_4",
            "test_cmd": "npm run test:unit -- --verbose --no-color"
          },
          "image_name": "docker.io/swerebenchv2/elastic-synthetics:316-f52f0bf",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Pinned issue, repair patch, test patch and environment configuration are available."
        ],
        "risks": [
          "Source added test exercises beforeAll; afterAll behavior needs a separate coverage review.",
          "Repository/image was not reconstructed or executed for this gallery."
        ],
        "must_hold": [
          "Lifecycle-hook failures remain visible to the reporting consumers.",
          "Unaffected journeys and successful runs preserve expected behavior."
        ],
        "may_vary": [
          "Error propagation structure and helper decomposition.",
          "Equivalent patch layouts and naming that preserve public behavior."
        ],
        "proposed_changes": [
          "Convert behavior and test obligations into criteria; accept different correct patches.",
          "Run failing/passing tests plus targeted uncovered cases before treating any probe as a verified repair."
        ]
      },
      "probes": [
        {
          "label": "Shared failure propagation sketch",
          "kind": "valid_alternative",
          "response": "Capture the beforeAll failure once, attach it to every affected journey result, and emit the normal reporter failure lifecycle for each journey.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "Potential valid implementation approach; requires runtime verification.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Result-construction sketch",
          "kind": "valid_alternative",
          "response": "Build failed journey results from a retained suite-hook error while centralizing reporter emission in the usual result path.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "Different internal organization can satisfy the same behavior.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Logs only",
          "kind": "invalid_control",
          "response": "Catch hook errors and print them, then return successful journey results.",
          "is_full_response": false,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "unexecuted_sketch",
          "why": "Consumers still observe success and lose the requested failure association.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "base_commit": "f52f0bf3d18ca418d1eec4afd1370751fdd914ce",
        "created_at": "2021-06-22 22:21:33",
        "image_name": "docker.io/swerebenchv2/elastic-synthetics:316-f52f0bf",
        "instance_id": "elastic__synthetics-316",
        "interface": "No new interfaces are introduced.",
        "language": "ts",
        "license": "MIT",
        "patch": "diff --git a/src/core/runner.ts b/src/core/runner.ts\nindex 2872cf5..649a22a 100644\n--- a/src/core/runner.ts\n+++ b/src/core/runner.ts\n@@ -131,6 +131,7 @@ export default class Runner extends EventEmitter {\n   journeys: Journey[] = [];\n   hooks: SuiteHooks = { beforeAll: [], afterAll: [] };\n   screenshotPath = join(CACHE_PATH, 'screenshots');\n+  hookError: Error | undefined;\n \n   static async createContext(options: RunOptions): Promise<JourneyContext> {\n     const start = monotonicTimeInSeconds();\n@@ -291,13 +292,12 @@ export default class Runner extends EventEmitter {\n     result: JourneyContext & JourneyResult,\n     options: RunOptions\n   ) {\n-    const { pluginManager, start, params, status, error } = result;\n+    const { pluginManager, start, status, error } = result;\n     const pluginOutput = await pluginManager.output();\n     this.emit('journey:end', {\n       journey,\n       status,\n       error,\n-      params,\n       start,\n       end: monotonicTimeInSeconds(),\n       options,\n@@ -313,6 +313,33 @@ export default class Runner extends EventEmitter {\n     }\n   }\n \n+  /**\n+   * Simulate a journey run to capture errors in the beforeAll hook\n+   */\n+  async runFakeJourney(journey: Journey, options: RunOptions) {\n+    const start = monotonicTimeInSeconds();\n+    this.emit('journey:start', {\n+      journey,\n+      timestamp: getTimestamp(),\n+      params: options.params,\n+    });\n+    const result: JourneyResult = {\n+      status: 'failed',\n+      error: this.hookError,\n+    };\n+    this.emit('journey:end', {\n+      journey,\n+      start,\n+      options,\n+      end: monotonicTimeInSeconds(),\n+      ...result,\n+    });\n+    if (options.reporter === 'json') {\n+      await once(this, 'journey:end:reported');\n+    }\n+    return result;\n+  }\n+\n   async runJourney(journey: Journey, options: RunOptions) {\n     const result: JourneyResult = {\n       status: 'succeeded',\n@@ -376,7 +403,8 @@ export default class Runner extends EventEmitter {\n     await this.runBeforeAllHook({\n       env: options.environment,\n       params: options.params,\n-    });\n+    }).catch(e => (this.hookError = e));\n+\n     const { dryRun, match, tags } = options;\n     for (const journey of this.journeys) {\n       /**\n@@ -389,7 +417,9 @@ export default class Runner extends EventEmitter {\n       if (!journey.isMatch(match, tags)) {\n         continue;\n       }\n-      const journeyResult = await this.runJourney(journey, options);\n+      const journeyResult: JourneyResult = this.hookError\n+        ? await this.runFakeJourney(journey, options)\n+        : await this.runJourney(journey, options);\n       result[journey.name] = journeyResult;\n     }\n     await Gatherer.stop();\ndiff --git a/src/reporters/json.ts b/src/reporters/json.ts\nindex f169df5..08d13bc 100644\n--- a/src/reporters/json.ts\n+++ b/src/reporters/json.ts\n@@ -332,7 +332,6 @@ export async function gatherScreenshots(\n   screenshotsPath: string,\n   callback: (step: Step, data: string) => Promise<void>\n ) {\n-  const screenshots: Array<ScreenshotOutput> = [];\n   if (isDirectory(screenshotsPath)) {\n     await totalist(screenshotsPath, async (_, absPath) => {\n       try {\n@@ -344,7 +343,6 @@ export async function gatherScreenshots(\n       }\n     });\n   }\n-  return screenshots;\n }\n \n export default class JSONReporter extends BaseReporter {\n@@ -425,6 +423,9 @@ export default class JSONReporter extends BaseReporter {\n           await gatherScreenshots(\n             join(CACHE_PATH, 'screenshots'),\n             async (step, data) => {\n+              if (!data) {\n+                return;\n+              }\n               if (ssblocks) {\n                 await this.writeScreenshotBlocks(journey, step, data);\n               } else {\n",
        "pr_description": "fix: capture beforeAll hook errors\n+ fix #280 \r\n+ We capture the beforeAll hook errors and any error that happens in any one of the hooks would be captured and all journeys that run on the current invocation will report that error event with `failed` status to be able to captured as part of the Uptime UI. \r\n+ `afterAll` errors behaves the same - we report them in the stderror logs which will be captured by Heartbeat. ",
        "problem_statement": "propagate errors from `beforeAll` and `afterAll` hooks\n+ Now error in the `beforeAll` and `afterAll` hooks would be captured and reported as error, but they will not be associated with the reporters in the correct way. \r\n+ Without associating these errors, the Uptime UI will have no information about what happened during the context of a single execution. \r\n\r\nWe have to figure out a way to solve this problem. ",
        "repo": "elastic/synthetics",
        "test_patch": "diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts\nindex a60b6ed..d4ba7d9 100644\n--- a/__tests__/core/runner.test.ts\n+++ b/__tests__/core/runner.test.ts\n@@ -147,6 +147,23 @@ describe('runner', () => {\n     });\n   });\n \n+  it('run journey - failed on beforeAll', async () => {\n+    const error = new Error('Broken beforeAll hook');\n+    runner.addHook('beforeAll', () => {\n+      throw error;\n+    });\n+    runner.addJourney(new Journey({ name: 'j1' }, () => step('step1', noop)));\n+    runner.addJourney(new Journey({ name: 'j2' }, () => step('step1', noop)));\n+    const result = await runner.run({\n+      wsEndpoint,\n+      outfd: fs.openSync(dest, 'w'),\n+    });\n+    expect(result).toEqual({\n+      j1: { status: 'failed', error },\n+      j2: { status: 'failed', error },\n+    });\n+  });\n+\n   it('run step', async () => {\n     const j1 = journey('j1', async ({ page }) => {\n       step('step1', async () => {\n",
        "FAIL_TO_PASS": [
          "run journey - failed on beforeAll"
        ],
        "PASS_TO_PASS": [
          "log to specified fd",
          "computes trace of the tab",
          "compute user timing metrics",
          "compute user experience trace and metrics",
          "computes layout shift",
          "computes cls with session window",
          "calculate cls score with simulated sessions",
          "cls to 0 when no events found",
          "computes filmstrips",
          "add step to the journey",
          "read config based on environment",
          "throw error when config does not exist",
          "recursively look for configs and exit",
          "indent message with seperator",
          "get monotonic clock time",
          "convert trace timestamp to internal time",
          "format errors",
          "throw error when no package.json found",
          "rewrite error stack from Playwright",
          "does not rewrite non playwright errors",
          "start plugin with given type",
          "get returns plugin instance",
          "stop plugin on output generation",
          "should capture page metrics",
          "should capture browser console logs",
          "should create browser pages",
          "should capture filmstrips",
          "boot and close browser",
          "setup and dispose driver",
          "begin recording based on flags",
          "should capture network info",
          "not include data URL in network info",
          "produce distinct events for redirects",
          "measure resource and transfer size",
          "timings for aborted requests",
          "timings for chunked response",
          "calculate timings for a request event",
          "when some resource timing data is unavailable",
          "when complete resource timing is not available",
          "writes the output to fd",
          "writes the output to a file",
          "writes each step to the FD",
          "render hook errors without steps",
          "multiple run invokes runner only once",
          "calls runner with proper options",
          "writes each step as NDJSON to the FD",
          "formats network fields in ECS format",
          "writes step errors to the top level",
          "writes journey errors to the top level",
          "writes full journey info if present",
          "captures number of journeys as metadata event",
          "return empty when dir doesnt exists",
          "idempotent on constructing screenshots blocks",
          "write whole blobs data",
          "write block & reference docs",
          "dont write on only-on-failure for successful journey",
          "write on only-on-failure for failed journey",
          "support emitting/subscribing to events",
          "add journeys",
          "add hooks",
          "run journey - with events payload",
          "run journey - failed when any step fails",
          "run journey - with hooks",
          "run journey - failed when hooks errors",
          "run step",
          "run step - syntax failure",
          "run step - navigation failure",
          "run step - bad navigation",
          "run steps - accumulate results",
          "run api",
          "run api - match journey name explict",
          "run api - match journey name and tag globs",
          "run api - prefer tags glob matching",
          "run api - support multiple tags",
          "run api - support negation tags",
          "run api - accumulate failed journeys",
          "run api - dry run",
          "run - should preserve order hooks/journeys/steps",
          "run - expose params in all hooks",
          "run - supports custom reporters",
          "run suites and exit with 1",
          "throw error on modifying params",
          "show warn for unknown capability flag"
        ],
        "install_config": {
          "base_image_name": "node_20",
          "docker_specs": null,
          "install": [
            "npm ci --quiet"
          ],
          "log_parser": "parse_log_js_4",
          "test_cmd": "npm run test:unit -- --verbose --no-color"
        },
        "meta": {
          "llm_metadata": {
            "code": "A",
            "confidence": 0.97,
            "detected_issues": {
              "B1": false,
              "B2": false,
              "B3": false,
              "B4": false,
              "B5": false,
              "B6": false
            },
            "difficulty": "medium",
            "external_urls": [],
            "intent_completeness": "complete",
            "pr_categories": [
              "minor_bug"
            ],
            "reasoning": "The issue requests that errors thrown in beforeAll/afterAll hooks be captured and cause all journeys to fail with the same error. The test adds a beforeAll hook that throws and expects each journey's result to be {status:'failed', error} with the same Error object. The test aligns with the described requirement and does not introduce unrelated expectations. No signals of B‑categories (no external URLs, naming constraints, ambiguous spec, etc.) are present, so the task is clearly solvable.",
            "test_alignment_issues": []
          },
          "num_modified_files": 2,
          "num_modified_lines": 37,
          "pr_author": "vigneshshanmugam",
          "pr_labels": [],
          "pr_url": null
        }
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "reference": {
        "answer": "Source gold repair patch (not executed in this gallery).",
        "solution": "diff --git a/src/core/runner.ts b/src/core/runner.ts\nindex 2872cf5..649a22a 100644\n--- a/src/core/runner.ts\n+++ b/src/core/runner.ts\n@@ -131,6 +131,7 @@ export default class Runner extends EventEmitter {\n   journeys: Journey[] = [];\n   hooks: SuiteHooks = { beforeAll: [], afterAll: [] };\n   screenshotPath = join(CACHE_PATH, 'screenshots');\n+  hookError: Error | undefined;\n \n   static async createContext(options: RunOptions): Promise<JourneyContext> {\n     const start = monotonicTimeInSeconds();\n@@ -291,13 +292,12 @@ export default class Runner extends EventEmitter {\n     result: JourneyContext & JourneyResult,\n     options: RunOptions\n   ) {\n-    const { pluginManager, start, params, status, error } = result;\n+    const { pluginManager, start, status, error } = result;\n     const pluginOutput = await pluginManager.output();\n     this.emit('journey:end', {\n       journey,\n       status,\n       error,\n-      params,\n       start,\n       end: monotonicTimeInSeconds(),\n       options,\n@@ -313,6 +313,33 @@ export default class Runner extends EventEmitter {\n     }\n   }\n \n+  /**\n+   * Simulate a journey run to capture errors in the beforeAll hook\n+   */\n+  async runFakeJourney(journey: Journey, options: RunOptions) {\n+    const start = monotonicTimeInSeconds();\n+    this.emit('journey:start', {\n+      journey,\n+      timestamp: getTimestamp(),\n+      params: options.params,\n+    });\n+    const result: JourneyResult = {\n+      status: 'failed',\n+      error: this.hookError,\n+    };\n+    this.emit('journey:end', {\n+      journey,\n+      start,\n+      options,\n+      end: monotonicTimeInSeconds(),\n+      ...result,\n+    });\n+    if (options.reporter === 'json') {\n+      await once(this, 'journey:end:reported');\n+    }\n+    return result;\n+  }\n+\n   async runJourney(journey: Journey, options: RunOptions) {\n     const result: JourneyResult = {\n       status: 'succeeded',\n@@ -376,7 +403,8 @@ export default class Runner extends EventEmitter {\n     await this.runBeforeAllHook({\n       env: options.environment,\n       params: options.params,\n-    });\n+    }).catch(e => (this.hookError = e));\n+\n     const { dryRun, match, tags } = options;\n     for (const journey of this.journeys) {\n       /**\n@@ -389,7 +417,9 @@ export default class Runner extends EventEmitter {\n       if (!journey.isMatch(match, tags)) {\n         continue;\n       }\n-      const journeyResult = await this.runJourney(journey, options);\n+      const journeyResult: JourneyResult = this.hookError\n+        ? await this.runFakeJourney(journey, options)\n+        : await this.runJourney(journey, options);\n       result[journey.name] = journeyResult;\n     }\n     await Gatherer.stop();\ndiff --git a/src/reporters/json.ts b/src/reporters/json.ts\nindex f169df5..08d13bc 100644\n--- a/src/reporters/json.ts\n+++ b/src/reporters/json.ts\n@@ -332,7 +332,6 @@ export async function gatherScreenshots(\n   screenshotsPath: string,\n   callback: (step: Step, data: string) => Promise<void>\n ) {\n-  const screenshots: Array<ScreenshotOutput> = [];\n   if (isDirectory(screenshotsPath)) {\n     await totalist(screenshotsPath, async (_, absPath) => {\n       try {\n@@ -344,7 +343,6 @@ export async function gatherScreenshots(\n       }\n     });\n   }\n-  return screenshots;\n }\n \n export default class JSONReporter extends BaseReporter {\n@@ -425,6 +423,9 @@ export default class JSONReporter extends BaseReporter {\n           await gatherScreenshots(\n             join(CACHE_PATH, 'screenshots'),\n             async (step, data) => {\n+              if (!data) {\n+                return;\n+              }\n               if (ssblocks) {\n                 await this.writeScreenshotBlocks(journey, step, data);\n               } else {\n",
        "tests": {
          "test_patch": "diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts\nindex a60b6ed..d4ba7d9 100644\n--- a/__tests__/core/runner.test.ts\n+++ b/__tests__/core/runner.test.ts\n@@ -147,6 +147,23 @@ describe('runner', () => {\n     });\n   });\n \n+  it('run journey - failed on beforeAll', async () => {\n+    const error = new Error('Broken beforeAll hook');\n+    runner.addHook('beforeAll', () => {\n+      throw error;\n+    });\n+    runner.addJourney(new Journey({ name: 'j1' }, () => step('step1', noop)));\n+    runner.addJourney(new Journey({ name: 'j2' }, () => step('step1', noop)));\n+    const result = await runner.run({\n+      wsEndpoint,\n+      outfd: fs.openSync(dest, 'w'),\n+    });\n+    expect(result).toEqual({\n+      j1: { status: 'failed', error },\n+      j2: { status: 'failed', error },\n+    });\n+  });\n+\n   it('run step', async () => {\n     const j1 = journey('j1', async ({ page }) => {\n       step('step1', async () => {\n",
          "FAIL_TO_PASS": [
            "run journey - failed on beforeAll"
          ],
          "PASS_TO_PASS": [
            "log to specified fd",
            "computes trace of the tab",
            "compute user timing metrics",
            "compute user experience trace and metrics",
            "computes layout shift",
            "computes cls with session window",
            "calculate cls score with simulated sessions",
            "cls to 0 when no events found",
            "computes filmstrips",
            "add step to the journey",
            "read config based on environment",
            "throw error when config does not exist",
            "recursively look for configs and exit",
            "indent message with seperator",
            "get monotonic clock time",
            "convert trace timestamp to internal time",
            "format errors",
            "throw error when no package.json found",
            "rewrite error stack from Playwright",
            "does not rewrite non playwright errors",
            "start plugin with given type",
            "get returns plugin instance",
            "stop plugin on output generation",
            "should capture page metrics",
            "should capture browser console logs",
            "should create browser pages",
            "should capture filmstrips",
            "boot and close browser",
            "setup and dispose driver",
            "begin recording based on flags",
            "should capture network info",
            "not include data URL in network info",
            "produce distinct events for redirects",
            "measure resource and transfer size",
            "timings for aborted requests",
            "timings for chunked response",
            "calculate timings for a request event",
            "when some resource timing data is unavailable",
            "when complete resource timing is not available",
            "writes the output to fd",
            "writes the output to a file",
            "writes each step to the FD",
            "render hook errors without steps",
            "multiple run invokes runner only once",
            "calls runner with proper options",
            "writes each step as NDJSON to the FD",
            "formats network fields in ECS format",
            "writes step errors to the top level",
            "writes journey errors to the top level",
            "writes full journey info if present",
            "captures number of journeys as metadata event",
            "return empty when dir doesnt exists",
            "idempotent on constructing screenshots blocks",
            "write whole blobs data",
            "write block & reference docs",
            "dont write on only-on-failure for successful journey",
            "write on only-on-failure for failed journey",
            "support emitting/subscribing to events",
            "add journeys",
            "add hooks",
            "run journey - with events payload",
            "run journey - failed when any step fails",
            "run journey - with hooks",
            "run journey - failed when hooks errors",
            "run step",
            "run step - syntax failure",
            "run step - navigation failure",
            "run step - bad navigation",
            "run steps - accumulate results",
            "run api",
            "run api - match journey name explict",
            "run api - match journey name and tag globs",
            "run api - prefer tags glob matching",
            "run api - support multiple tags",
            "run api - support negation tags",
            "run api - accumulate failed journeys",
            "run api - dry run",
            "run - should preserve order hooks/journeys/steps",
            "run - expose params in all hooks",
            "run - supports custom reporters",
            "run suites and exit with 1",
            "throw error on modifying params",
            "show warn for unknown capability flag"
          ]
        },
        "warning": "Repository tests and the gold patch have not been executed here. Response probes are implementation sketches."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-swe-rebench-v2-wtforms__wtforms-614",
      "title": "Make HTML5 widgets the default while preserving interfaces",
      "source": {
        "dataset": "nebius/SWE-rebench-V2",
        "url": "https://huggingface.co/datasets/nebius/SWE-rebench-V2/blob/475dd5e8703bb5fb22dd3c60b5d038b019eba1e0/data/train-00000-of-00001.parquet",
        "revision": "475dd5e8703bb5fb22dd3c60b5d038b019eba1e0",
        "split": "train",
        "row_id": "wtforms__wtforms-614",
        "row_index_in_file": 1,
        "source_file": "data/train-00000-of-00001.parquet",
        "license": "CC-BY-4.0 dataset; original repository BSD-3-Clause",
        "pool": "proposed",
        "row_sha256": "2cc95684f12ff757412f38b0f7a3f2da8afce7998f8133f3e865f3b22acc8b97",
        "attribution": "Source task and reference: nebius/SWE-rebench-V2. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Original repository: wtforms/wtforms (BSD-3-Clause).",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/nebius/SWE-rebench-V2/blob/475dd5e8703bb5fb22dd3c60b5d038b019eba1e0/data/train-00000-of-00001.parquet"
          },
          {
            "label": "Original repository at task base",
            "url": "https://github.com/wtforms/wtforms/tree/848d28d67e45cda7a06c4c8ed2768e6a8cb1c016"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "software_repair",
        "leaf": "software_repair.interface_compatibility_repair",
        "confidence": "editorial_high",
        "reason": "Requires modifying an existing repository at a pinned commit, not writing a generic code answer."
      },
      "language": "en",
      "prompt": "Use HTML 5 widgets\nThe HTML 5 widgets are kept in their own section and not used by default, but this distinction doesn't make sense today. \"HTML 5\" is just \"HTML\" now. No supported browser doesn't understand those input types, and ones that aren't supported fall back to text anyway.\r\n\r\nI already sort of started this by making the required validators add the `required` flag, and I'm taking it further with #406 for more complex flags. Fields should default to using the more specific widgets where possible.",
      "context": [
        "Repository: wtforms/wtforms\nBase commit: 848d28d67e45cda7a06c4c8ed2768e6a8cb1c016",
        "Source interface contract:\nMethod: Flags.__getattr__(self, name)  \nLocation: wtforms/fields/core.py → class Flags  \nInputs: *name* – attribute name requested on a Flags instance.  \nOutputs: Returns the attribute’s value if it exists; otherwise returns **None** (previously returned False).  \nDescription: Provides a unified flags object where missing flags evaluate to None, enabling flag‑based widget rendering.\n\nMethod: Input.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class Input  \nInputs: *field* – a WTForms Field instance; **kwargs – optional HTML attributes passed at render time.  \nOutputs: A Markup string for an `<input …>` element; automatically injects any flag attributes whose names appear in the widget’s *validation_attrs* list and are set on *field.flags*.  \nDescription: Renders HTML5‑compatible input elements, pulling validation‑related attributes (e.g., required, minlength, maxlength, min, max, step) from the field’s flags.\n\nMethod: TextInput.__call__(self, field, **kwargs) *(inherited, uses Input.__call__)*  \nLocation: wtforms/widgets/core.py → class TextInput  \nInputs: Same as Input.__call__.  \nOutputs: `<input type=\"text\" …>` markup with added validation attributes.  \nDescription: Text input widget that now supports required, maxlength, minlength, and pattern attributes derived from field flags.\n\nMethod: PasswordInput.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class PasswordInput  \nInputs: Same as Input.__call__.  \nOutputs: `<input type=\"password\" …>` markup with validation attributes.  \nDescription: Password input widget supporting the same validation flags as TextInput.\n\nMethod: TextArea.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class TextArea  \nInputs: *field*; **kwargs.  \nOutputs: `<textarea …>` markup with required, maxlength, minlength attributes taken from flags.  \nDescription: Textarea widget now propagates flag‑based validation attributes.\n\nMethod: Select.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class Select  \nInputs: *field*; **kwargs.  \nOutputs: `<select …>` markup with required attribute from flags.  \nDescription: Select widget now adds a required attribute when the field’s flags indicate it.\n\nMethod: SearchField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class SearchField (inherits StringField)  \nInputs: *label*, *validators*, additional field kwargs.  \nOutputs: Instance of a field rendering `<input type=\"search\">`.  \nDescription: New convenience field for HTML `search` input type, using the SearchInput widget.\n\nMethod: TelField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class TelField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"tel\">`.  \nDescription: New field for telephone inputs.\n\nMethod: URLField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class URLField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"url\">`.  \nDescription: New field for URL inputs.\n\nMethod: EmailField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class EmailField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"email\">`.  \nDescription: New field for email inputs.\n\nMethod: DateTimeLocalField.__init__(self, label=None, validators=None, format=\"%Y-%m-%d %H:%M:%S\", **kwargs)  \nLocation: wtforms/fields/core.py → class DateTimeLocalField (inherits DateTimeField)  \nInputs: *label*, *validators*, *format*, additional kwargs.  \nOutputs: Instance rendering `<input type=\"datetime-local\">`.  \nDescription: New field for HTML5 datetime‑local inputs.\n\nMethod: IntegerRangeField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class IntegerRangeField (inherits IntegerField)  \nInputs: Same as IntegerField.  \nOutputs: Instance rendering `<input type=\"range\">`.  \nDescription: New field for numeric range inputs, using the RangeInput widget.\n\nMethod: DecimalRangeField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class DecimalRangeField (inherits DecimalField)  \nInputs: Same as DecimalField.  \nOutputs: Instance rendering `<input type=\"range\">` with step=\"any\".  \nDescription: New field for decimal range inputs.\n\nMethod: Length.__init__(self, min=-1, max=-1, message=None)  \nLocation: wtforms/validators.py → class Length  \nInputs: *min* (int, default –1), *max* (int, default –1), *message* (optional).  \nOutputs: Validator instance with *field_flags* dict containing “minlength” and/or “maxlength” when limits are set.  \nDescription: Length validator now provides flag information for widget rendering.\n\nMethod: NumberRange.__init__(self, min=None, max=None, message=None)  \nLocation: wtforms/validators.py → class NumberRange  \nInputs: *min* (numeric or None), *max* (numeric or None), *message*.  \nOutputs: Validator instance with *field_flags* dict containing “min” and/or “max”.  \nDescription: NumberRange validator now supplies min/max flags for HTML5 widgets.\n\nMethod: Optional.__init__(self, strip_whitespace=True)  \nLocation: wtforms/validators.py → class Optional  \nInputs: *strip_whitespace* (bool).  \nOutputs: Validator instance with *field_flags* = {\"optional\": True}.  \nDescription: Optional validator now sets an explicit “optional” flag for widgets.\n\nMethod: DataRequired.__init__(self, message=None)  \nLocation: wtforms/validators.py → class DataRequired  \nInputs: *message* (optional).  \nOutputs: Validator instance with *field_flags* = {\"required\": True}.  \nDescription: DataRequired now provides a required flag for widget rendering.\n\nMethod: InputRequired.__init__(self, message=None)  \nLocation: wtforms/validators.py → class InputRequired  \nInputs: *message* (optional).  \nOutputs: Validator instance with *field_flags* = {\"required\": True}.  \nDescription: InputRequired now provides the required flag similarly to DataRequired.\n\nFunction: html_params(**kwargs) *(unchanged signature, but now used by widgets to include flag attributes)*  \nLocation: wtforms/widgets/core.py → helper function.  \nInputs: Arbitrary HTML attribute key/value pairs.  \nOutputs: Properly escaped attribute string.  \nDescription: Generates the attribute list for widget rendering, now receives additional validation attributes from flags.\n\n"
      ],
      "environment_note": "Requires reconstruction of wtforms/wtforms at 848d28d67e45cda7a06c4c8ed2768e6a8cb1c016 with the supplied source environment. This gallery has not run the repository tests.",
      "rubric": {
        "markdown": "# Make HTML5 widgets the default while preserving interfaces\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Requested default behavior\n\n**ID:** c1\n\n**Criterion:** Use the requested HTML5-compatible widgets/fields by default and propagate relevant validation attributes according to the supplied issue/interface contract.\n\n**Points:** +6\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Behavior\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Public interface and compatibility\n\n**ID:** c2\n\n**Criterion:** Keep documented imports and expected field/widget interactions coherent; distinguish intended documented changes, such as missing flag values, from unintended breakage.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Compatibility\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Executable regression evidence\n\n**ID:** c3\n\n**Criterion:** Produce an applicable patch at the base revision and pass the relevant source tests plus normal field/widget regressions. Equivalent internal organization is allowed; the reference patch is not mandatory.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n",
        "raw": {
          "gold_patch": "diff --git a/CHANGES.rst b/CHANGES.rst\nindex 50f43a2..a199f46 100644\n--- a/CHANGES.rst\n+++ b/CHANGES.rst\n@@ -34,7 +34,7 @@ Unreleased\n -   Render attribute names like ``for_`` and ``class_`` are normalized\n     consistently so later values override those specified earlier.\n     :issue:`449`, :pr:`596`\n-\n+-   Flags can take non-boolean values. :issue:`406` :pr:`467`\n \n Version 2.3.1\n -------------\ndiff --git a/docs/fields.rst b/docs/fields.rst\nindex ed047ce..0bd3e32 100644\n--- a/docs/fields.rst\n+++ b/docs/fields.rst\n@@ -182,10 +182,10 @@ The Field base class\n \n     .. attribute:: flags\n \n-        An object containing boolean flags set either by the field itself, or\n+        An object containing flags set either by the field itself, or\n         by validators on the field. For example, the built-in\n         :class:`~wtforms.validators.InputRequired` validator sets the `required` flag.\n-        An unset flag will result in :const:`False`.\n+        An unset flag will result in :const:`None`.\n \n         .. code-block:: html+django\n \n@@ -220,10 +220,14 @@ refer to a single input from the form.\n \n .. autoclass:: DateTimeField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n-    For better date/time fields, see the :mod:`dateutil extension <wtforms.ext.dateutil.fields>`\n+.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n .. autoclass:: DecimalField(default field arguments, places=2, rounding=None, use_locale=False, number_format=None)\n \n+.. autoclass:: DecimalRangeField(default field arguments)\n+\n+.. autoclass:: EmailField(default field arguments)\n+\n .. autoclass:: FileField(default field arguments)\n \n     Example usage::\n@@ -252,6 +256,8 @@ refer to a single input from the form.\n \n .. autoclass:: IntegerField(default field arguments)\n \n+.. autoclass:: IntegerRangeField(default field arguments)\n+\n .. autoclass:: RadioField(default field arguments, choices=[], coerce=str)\n \n     .. code-block:: jinja\n@@ -323,6 +329,8 @@ refer to a single input from the form.\n     a list of fields each representing an option. The rendering of this can be\n     further controlled by specifying `option_widget=`.\n \n+.. autoclass:: SearchField(default field arguments)\n+\n .. autoclass:: SelectMultipleField(default field arguments, choices=[], coerce=str, option_widget=None)\n \n    The data on the SelectMultipleField is stored as a list of objects, each of\n@@ -338,6 +346,13 @@ refer to a single input from the form.\n \n         {{ form.username(size=30, maxlength=50) }}\n \n+.. autoclass:: TelField(default field arguments)\n+\n+.. autoclass:: TimeField(default field arguments, format='%H:%M')\n+\n+.. autoclass:: URLField(default field arguments)\n+\n+\n Convenience Fields\n ------------------\n \n@@ -559,40 +574,3 @@ Additional Helper Classes\n     .. attribute:: text\n \n         The original label text passed to the field's constructor.\n-\n-HTML5 Fields\n-------------\n-\n-In addition to basic HTML fields, WTForms also supplies fields for the HTML5\n-standard. These fields can be accessed under the :mod:`wtforms.fields.html5` namespace.\n-In reality, these fields are just convenience fields that extend basic fields\n-and implement HTML5 specific widgets. These widgets are located in the :mod:`wtforms.widgets.html5`\n-namespace and can be overridden or modified just like any other widget.\n-\n-.. module:: wtforms.fields.html5\n-\n-.. autoclass:: SearchField(default field arguments)\n-\n-.. autoclass:: TelField(default field arguments)\n-\n-.. autoclass:: URLField(default field arguments)\n-\n-.. autoclass:: EmailField(default field arguments)\n-\n-.. autoclass:: DateTimeField(default field arguments,  format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: DateField(default field arguments, format='%Y-%m-%d')\n-\n-.. autoclass:: MonthField(default field arguments, format='%Y-%m')\n-\n-.. autoclass:: TimeField(default field arguments, format='%H:%M')\n-\n-.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: IntegerField(default field arguments)\n-\n-.. autoclass:: DecimalField(default field arguments)\n-\n-.. autoclass:: IntegerRangeField(default field arguments)\n-\n-.. autoclass:: DecimalRangeField(default field arguments)\ndiff --git a/docs/validators.rst b/docs/validators.rst\nindex cdb9627..818c018 100644\n--- a/docs/validators.rst\n+++ b/docs/validators.rst\n@@ -177,22 +177,21 @@ which will then be available on the :attr:`field's flags object\n \n To specify flags on your validator, set the ``field_flags`` attribute on your\n validator. When the Field is constructed, the flags with the same name will be\n-set to True on your field. For example, let's imagine a validator that\n+set on your field. For example, let's imagine a validator that\n validates that input is valid BBCode. We can set a flag on the field then to\n signify that the field accepts BBCode::\n \n     # class implementation\n     class ValidBBCode(object):\n-        field_flags = ('accepts_bbcode', )\n-\n-        pass # validator implementation here\n+        def __init__(self):\n+            self.field_flags = {'accepts_bbcode': True}\n \n     # factory implementation\n     def valid_bbcode():\n         def _valid_bbcode(form, field):\n             pass # validator implementation here\n \n-        _valid_bbcode.field_flags = ('accepts_bbcode', )\n+        _valid_bbcode.field_flags = {'accepts_bbcode': True}\n         return _valid_bbcode\n \n Then we can check it in our template, so we can then place a note to the user:\n@@ -206,7 +205,10 @@ Then we can check it in our template, so we can then place a note to the user:\n \n Some considerations on using flags:\n \n-* Flags can only set boolean values, and another validator cannot unset them.\n-* If multiple fields set the same flag, its value is still True.\n+* Boolean flags will set HTML valueless attributes (e.g. `{required: True}`\n+  will give `<input type=\"text\" required>`). Other flag types will set regular\n+  HTML attributes (e.g. `{maxlength: 8}` will give `<input type=\"text\" maxlength=\"8\">`).\n+* If multiple validators set the same flag, the flag will have the value set\n+  by the first validator.\n * Flags are set from validators only in :meth:`Field.__init__`, so inline\n   validators and extra passed-in validators cannot set them.\ndiff --git a/docs/widgets.rst b/docs/widgets.rst\nindex 81b96c0..5c4f009 100644\n--- a/docs/widgets.rst\n+++ b/docs/widgets.rst\n@@ -14,36 +14,30 @@ recognize as not needing to be auto-escaped.\n Built-in widgets\n ----------------\n \n-.. autoclass:: wtforms.widgets.ListWidget\n-.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.ColorInput\n+.. autoclass:: wtforms.widgets.CheckboxInput\n+.. autoclass:: wtforms.widgets.DateTimeInput\n+.. autoclass:: wtforms.widgets.DateTimeLocalInput\n+.. autoclass:: wtforms.widgets.DateInput\n+.. autoclass:: wtforms.widgets.EmailInput\n+.. autoclass:: wtforms.widgets.FileInput\n+.. autoclass:: wtforms.widgets.HiddenInput\n .. autoclass:: wtforms.widgets.Input\n-.. autoclass:: wtforms.widgets.TextInput()\n+.. autoclass:: wtforms.widgets.ListWidget\n+.. autoclass:: wtforms.widgets.MonthInput\n+.. autoclass:: wtforms.widgets.NumberInput\n .. autoclass:: wtforms.widgets.PasswordInput\n-.. autoclass:: wtforms.widgets.HiddenInput()\n-.. autoclass:: wtforms.widgets.CheckboxInput()\n-.. autoclass:: wtforms.widgets.FileInput()\n-.. autoclass:: wtforms.widgets.SubmitInput()\n-.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.RangeInput\n+.. autoclass:: wtforms.widgets.SubmitInput\n+.. autoclass:: wtforms.widgets.SearchInput\n .. autoclass:: wtforms.widgets.Select\n-\n-HTML5 widgets\n--------------\n-\n-.. module:: wtforms.widgets.html5\n-\n-.. autoclass:: wtforms.widgets.html5.ColorInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeLocalInput\n-.. autoclass:: wtforms.widgets.html5.DateInput\n-.. autoclass:: wtforms.widgets.html5.EmailInput\n-.. autoclass:: wtforms.widgets.html5.MonthInput\n-.. autoclass:: wtforms.widgets.html5.NumberInput\n-.. autoclass:: wtforms.widgets.html5.RangeInput\n-.. autoclass:: wtforms.widgets.html5.SearchInput\n-.. autoclass:: wtforms.widgets.html5.TelInput\n-.. autoclass:: wtforms.widgets.html5.TimeInput\n-.. autoclass:: wtforms.widgets.html5.URLInput\n-.. autoclass:: wtforms.widgets.html5.WeekInput\n+.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.TelInput\n+.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.TextInput\n+.. autoclass:: wtforms.widgets.TimeInput\n+.. autoclass:: wtforms.widgets.URLInput\n+.. autoclass:: wtforms.widgets.WeekInput\n \n Widget-Building Utilities\n -------------------------\ndiff --git a/src/wtforms/fields/__init__.py b/src/wtforms/fields/__init__.py\nindex 1f770d3..e429631 100644\n--- a/src/wtforms/fields/__init__.py\n+++ b/src/wtforms/fields/__init__.py\n@@ -3,6 +3,5 @@ from wtforms.fields.core import Field\n from wtforms.fields.core import Flags\n from wtforms.fields.core import Label\n from wtforms.fields.core import SelectFieldBase\n-from wtforms.fields.html5 import *\n from wtforms.fields.simple import *\n from wtforms.utils import unset_value as _unset_value\ndiff --git a/src/wtforms/fields/core.py b/src/wtforms/fields/core.py\nindex d835092..8724356 100644\n--- a/src/wtforms/fields/core.py\n+++ b/src/wtforms/fields/core.py\n@@ -2,6 +2,7 @@ import datetime\n import decimal\n import inspect\n import itertools\n+import warnings\n \n from markupsafe import escape\n from markupsafe import Markup\n@@ -15,18 +16,25 @@ from wtforms.validators import ValidationError\n __all__ = (\n     \"BooleanField\",\n     \"DecimalField\",\n+    \"DecimalRangeField\",\n     \"DateField\",\n     \"DateTimeField\",\n+    \"DateTimeLocalField\",\n+    \"EmailField\",\n     \"FieldList\",\n     \"FloatField\",\n     \"FormField\",\n     \"IntegerField\",\n+    \"IntegerRangeField\",\n     \"RadioField\",\n+    \"SearchField\",\n     \"SelectField\",\n     \"SelectMultipleField\",\n     \"StringField\",\n+    \"TelField\",\n     \"TimeField\",\n     \"MonthField\",\n+    \"URLField\",\n )\n \n \n@@ -143,9 +151,21 @@ class Field:\n             self.widget = widget\n \n         for v in itertools.chain(self.validators, [self.widget]):\n-            flags = getattr(v, \"field_flags\", ())\n-            for f in flags:\n-                setattr(self.flags, f, True)\n+            flags = getattr(v, \"field_flags\", {})\n+\n+            # check for legacy format, remove eventually\n+            if isinstance(flags, tuple):\n+                warnings.warn(\n+                    \"Flags should be stored in dicts and not in tuples. \"\n+                    \"The next version of WTForms will abandon support \"\n+                    \"for flags in tuples.\",\n+                    DeprecationWarning,\n+                    stacklevel=2,\n+                )\n+                flags = {flag_name: True for flag_name in flags}\n+\n+            for k, v in flags.items():\n+                setattr(self.flags, k, v)\n \n     def __str__(self):\n         \"\"\"\n@@ -408,15 +428,15 @@ class UnboundField:\n \n class Flags:\n     \"\"\"\n-    Holds a set of boolean flags as attributes.\n+    Holds a set of flags as attributes.\n \n-    Accessing a non-existing attribute returns False for its value.\n+    Accessing a non-existing attribute returns None for its value.\n     \"\"\"\n \n     def __getattr__(self, name):\n         if name.startswith(\"_\"):\n             return super().__getattr__(name)\n-        return False\n+        return None\n \n     def __contains__(self, name):\n         return getattr(self, name)\n@@ -670,7 +690,7 @@ class IntegerField(Field):\n     is ignored and will not be accepted as a value.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput()\n \n     def __init__(self, label=None, validators=None, **kwargs):\n         super().__init__(label, validators, **kwargs)\n@@ -721,7 +741,7 @@ class DecimalField(LocaleAwareNumberField):\n         format for the locale.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput(step=\"any\")\n \n     def __init__(\n         self, label=None, validators=None, places=unset_value, rounding=None, **kwargs\n@@ -842,7 +862,7 @@ class DateTimeField(Field):\n     A text field which stores a `datetime.datetime` matching a format.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.DateTimeInput()\n \n     def __init__(\n         self, label=None, validators=None, format=\"%Y-%m-%d %H:%M:%S\", **kwargs\n@@ -871,6 +891,8 @@ class DateField(DateTimeField):\n     Same as DateTimeField, except stores a `datetime.date`.\n     \"\"\"\n \n+    widget = widgets.DateInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m-%d\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -889,6 +911,8 @@ class TimeField(DateTimeField):\n     Same as DateTimeField, except stores a `time`.\n     \"\"\"\n \n+    widget = widgets.TimeInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%H:%M\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -908,6 +932,8 @@ class MonthField(DateField):\n     with `day = 1`.\n     \"\"\"\n \n+    widget = widgets.MonthInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -1189,3 +1215,59 @@ class FieldList(Field):\n     @property\n     def data(self):\n         return [f.data for f in self.entries]\n+\n+\n+class SearchField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"search\">``.\n+    \"\"\"\n+\n+    widget = widgets.SearchInput()\n+\n+\n+class TelField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"tel\">``.\n+    \"\"\"\n+\n+    widget = widgets.TelInput()\n+\n+\n+class URLField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"url\">``.\n+    \"\"\"\n+\n+    widget = widgets.URLInput()\n+\n+\n+class EmailField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"email\">``.\n+    \"\"\"\n+\n+    widget = widgets.EmailInput()\n+\n+\n+class DateTimeLocalField(DateTimeField):\n+    \"\"\"\n+    Represents an ``<input type=\"datetime-local\">``.\n+    \"\"\"\n+\n+    widget = widgets.DateTimeLocalInput()\n+\n+\n+class IntegerRangeField(IntegerField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput()\n+\n+\n+class DecimalRangeField(DecimalField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/fields/html5.py b/src/wtforms/fields/html5.py\ndeleted file mode 100644\nindex 26d7efa..0000000\n--- a/src/wtforms/fields/html5.py\n+++ /dev/null\n@@ -1,121 +0,0 @@\n-from . import core\n-from ..widgets import html5 as widgets\n-\n-__all__ = (\n-    \"DateField\",\n-    \"DateTimeField\",\n-    \"DateTimeLocalField\",\n-    \"DecimalField\",\n-    \"DecimalRangeField\",\n-    \"EmailField\",\n-    \"IntegerField\",\n-    \"IntegerRangeField\",\n-    \"SearchField\",\n-    \"TelField\",\n-    \"TimeField\",\n-    \"URLField\",\n-)\n-\n-\n-class SearchField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"search\">``.\n-    \"\"\"\n-\n-    widget = widgets.SearchInput()\n-\n-\n-class TelField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"tel\">``.\n-    \"\"\"\n-\n-    widget = widgets.TelInput()\n-\n-\n-class URLField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"url\">``.\n-    \"\"\"\n-\n-    widget = widgets.URLInput()\n-\n-\n-class EmailField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"email\">``.\n-    \"\"\"\n-\n-    widget = widgets.EmailInput()\n-\n-\n-class DateTimeField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeInput()\n-\n-\n-class DateField(core.DateField):\n-    \"\"\"\n-    Represents an ``<input type=\"date\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateInput()\n-\n-\n-class TimeField(core.TimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"time\">``.\n-    \"\"\"\n-\n-    widget = widgets.TimeInput()\n-\n-\n-class MonthField(core.MonthField):\n-    \"\"\"\n-    Represents an ``<input type=\"month\">``.\n-    \"\"\"\n-\n-    widget = widgets.MonthInput()\n-\n-\n-class DateTimeLocalField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime-local\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeLocalInput()\n-\n-\n-class IntegerField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput()\n-\n-\n-class DecimalField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput(step=\"any\")\n-\n-\n-class IntegerRangeField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput()\n-\n-\n-class DecimalRangeField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/validators.py b/src/wtforms/validators.py\nindex 2d6bea2..059f32c 100644\n--- a/src/wtforms/validators.py\n+++ b/src/wtforms/validators.py\n@@ -114,6 +114,8 @@ class Length:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)d` and `%(max)d` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `minlength` and `maxlength` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=-1, max=-1, message=None):\n@@ -124,6 +126,11 @@ class Length:\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min != -1:\n+            self.field_flags[\"minlength\"] = self.min\n+        if self.max != -1:\n+            self.field_flags[\"maxlength\"] = self.max\n \n     def __call__(self, form, field):\n         length = field.data and len(field.data) or 0\n@@ -174,12 +181,19 @@ class NumberRange:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)s` and `%(max)s` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `min` and `max` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=None, max=None, message=None):\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min is not None:\n+            self.field_flags[\"min\"] = self.min\n+        if self.max is not None:\n+            self.field_flags[\"max\"] = self.max\n \n     def __call__(self, form, field):\n         data = field.data\n@@ -215,9 +229,9 @@ class Optional:\n     :param strip_whitespace:\n         If True (the default) also stop the validation chain on input which\n         consists of only whitespace.\n-    \"\"\"\n \n-    field_flags = (\"optional\",)\n+    Sets the `optional` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, strip_whitespace=True):\n         if strip_whitespace:\n@@ -225,6 +239,8 @@ class Optional:\n         else:\n             self.string_check = lambda s: s\n \n+        self.field_flags = {\"optional\": True}\n+\n     def __call__(self, form, field):\n         if (\n             not field.raw_data\n@@ -256,12 +272,13 @@ class DataRequired:\n \n     :param message:\n         Error message to raise in case of a validation error.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.data or isinstance(field.data, str) and not field.data.strip():\n@@ -281,12 +298,13 @@ class InputRequired:\n     Note there is a distinction between this and DataRequired in that\n     InputRequired looks that form-input data was provided, and DataRequired\n     looks at the post-coercion data.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.raw_data or not field.raw_data[0]:\ndiff --git a/src/wtforms/widgets/core.py b/src/wtforms/widgets/core.py\nindex 3e2a2eb..a0e0e61 100644\n--- a/src/wtforms/widgets/core.py\n+++ b/src/wtforms/widgets/core.py\n@@ -3,17 +3,30 @@ from markupsafe import Markup\n \n __all__ = (\n     \"CheckboxInput\",\n+    \"ColorInput\",\n+    \"DateInput\",\n+    \"DateTimeInput\",\n+    \"DateTimeLocalInput\",\n+    \"EmailInput\",\n     \"FileInput\",\n     \"HiddenInput\",\n     \"ListWidget\",\n+    \"MonthInput\",\n+    \"NumberInput\",\n+    \"Option\",\n     \"PasswordInput\",\n     \"RadioInput\",\n+    \"RangeInput\",\n+    \"SearchInput\",\n     \"Select\",\n     \"SubmitInput\",\n     \"TableWidget\",\n     \"TextArea\",\n     \"TextInput\",\n-    \"Option\",\n+    \"TelInput\",\n+    \"TimeInput\",\n+    \"URLInput\",\n+    \"WeekInput\",\n )\n \n \n@@ -148,6 +161,7 @@ class Input:\n     \"\"\"\n \n     html_params = staticmethod(html_params)\n+    validation_attrs = [\"required\"]\n \n     def __init__(self, input_type=None):\n         if input_type is not None:\n@@ -158,8 +172,10 @@ class Input:\n         kwargs.setdefault(\"type\", self.input_type)\n         if \"value\" not in kwargs:\n             kwargs[\"value\"] = field._value()\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\"<input %s>\" % self.html_params(name=field.name, **kwargs))\n \n \n@@ -169,6 +185,7 @@ class TextInput(Input):\n     \"\"\"\n \n     input_type = \"text\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n \n class PasswordInput(Input):\n@@ -181,6 +198,7 @@ class PasswordInput(Input):\n     \"\"\"\n \n     input_type = \"password\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n     def __init__(self, hide_value=True):\n         self.hide_value = hide_value\n@@ -196,9 +214,12 @@ class HiddenInput(Input):\n     Render a hidden input.\n     \"\"\"\n \n-    field_flags = (\"hidden\",)\n     input_type = \"hidden\"\n \n+    def __init__(self, *args, **kwargs):\n+        super().__init__(*args, **kwargs)\n+        self.field_flags = {\"hidden\": True}\n+\n \n class CheckboxInput(Input):\n     \"\"\"\n@@ -238,6 +259,7 @@ class FileInput(Input):\n     \"\"\"\n \n     input_type = \"file\"\n+    validation_attrs = [\"required\", \"accept\"]\n \n     def __init__(self, multiple=False):\n         super().__init__()\n@@ -275,10 +297,14 @@ class TextArea:\n     `rows` and `cols` ought to be passed as keyword args when rendering.\n     \"\"\"\n \n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\"]\n+\n     def __call__(self, field, **kwargs):\n         kwargs.setdefault(\"id\", field.id)\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\n             \"<textarea %s>\\r\\n%s</textarea>\"\n             % (html_params(name=field.name, **kwargs), escape(field._value()))\n@@ -297,6 +323,8 @@ class Select:\n     `(value, label, selected)`.\n     \"\"\"\n \n+    validation_attrs = [\"required\"]\n+\n     def __init__(self, multiple=False):\n         self.multiple = multiple\n \n@@ -304,8 +332,10 @@ class Select:\n         kwargs.setdefault(\"id\", field.id)\n         if self.multiple:\n             kwargs[\"multiple\"] = True\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         html = [\"<select %s>\" % html_params(name=field.name, **kwargs)]\n         for val, label, selected in field.iter_choices():\n             html.append(self.render_option(val, label, selected))\n@@ -338,3 +368,141 @@ class Option:\n         return Select.render_option(\n             field._value(), field.label.text, field.checked, **kwargs\n         )\n+\n+\n+class SearchInput(Input):\n+    \"\"\"\n+    Renders an input with type \"search\".\n+    \"\"\"\n+\n+    input_type = \"search\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class TelInput(Input):\n+    \"\"\"\n+    Renders an input with type \"tel\".\n+    \"\"\"\n+\n+    input_type = \"tel\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class URLInput(Input):\n+    \"\"\"\n+    Renders an input with type \"url\".\n+    \"\"\"\n+\n+    input_type = \"url\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class EmailInput(Input):\n+    \"\"\"\n+    Renders an input with type \"email\".\n+    \"\"\"\n+\n+    input_type = \"email\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class DateTimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime\".\n+    \"\"\"\n+\n+    input_type = \"datetime\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateInput(Input):\n+    \"\"\"\n+    Renders an input with type \"date\".\n+    \"\"\"\n+\n+    input_type = \"date\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class MonthInput(Input):\n+    \"\"\"\n+    Renders an input with type \"month\".\n+    \"\"\"\n+\n+    input_type = \"month\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class WeekInput(Input):\n+    \"\"\"\n+    Renders an input with type \"week\".\n+    \"\"\"\n+\n+    input_type = \"week\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class TimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"time\".\n+    \"\"\"\n+\n+    input_type = \"time\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateTimeLocalInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime-local\".\n+    \"\"\"\n+\n+    input_type = \"datetime-local\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class NumberInput(Input):\n+    \"\"\"\n+    Renders an input with type \"number\".\n+    \"\"\"\n+\n+    input_type = \"number\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None, min=None, max=None):\n+        self.step = step\n+        self.min = min\n+        self.max = max\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        if self.min is not None:\n+            kwargs.setdefault(\"min\", self.min)\n+        if self.max is not None:\n+            kwargs.setdefault(\"max\", self.max)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class RangeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"range\".\n+    \"\"\"\n+\n+    input_type = \"range\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None):\n+        self.step = step\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class ColorInput(Input):\n+    \"\"\"\n+    Renders an input with type \"color\".\n+    \"\"\"\n+\n+    input_type = \"color\"\ndiff --git a/src/wtforms/widgets/html5.py b/src/wtforms/widgets/html5.py\ndeleted file mode 100644\nindex 2a9ea76..0000000\n--- a/src/wtforms/widgets/html5.py\n+++ /dev/null\n@@ -1,143 +0,0 @@\n-from .core import Input\n-\n-__all__ = (\n-    \"ColorInput\",\n-    \"DateInput\",\n-    \"DateTimeInput\",\n-    \"DateTimeLocalInput\",\n-    \"EmailInput\",\n-    \"MonthInput\",\n-    \"NumberInput\",\n-    \"RangeInput\",\n-    \"SearchInput\",\n-    \"TelInput\",\n-    \"TimeInput\",\n-    \"URLInput\",\n-    \"WeekInput\",\n-)\n-\n-\n-class SearchInput(Input):\n-    \"\"\"\n-    Renders an input with type \"search\".\n-    \"\"\"\n-\n-    input_type = \"search\"\n-\n-\n-class TelInput(Input):\n-    \"\"\"\n-    Renders an input with type \"tel\".\n-    \"\"\"\n-\n-    input_type = \"tel\"\n-\n-\n-class URLInput(Input):\n-    \"\"\"\n-    Renders an input with type \"url\".\n-    \"\"\"\n-\n-    input_type = \"url\"\n-\n-\n-class EmailInput(Input):\n-    \"\"\"\n-    Renders an input with type \"email\".\n-    \"\"\"\n-\n-    input_type = \"email\"\n-\n-\n-class DateTimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime\".\n-    \"\"\"\n-\n-    input_type = \"datetime\"\n-\n-\n-class DateInput(Input):\n-    \"\"\"\n-    Renders an input with type \"date\".\n-    \"\"\"\n-\n-    input_type = \"date\"\n-\n-\n-class MonthInput(Input):\n-    \"\"\"\n-    Renders an input with type \"month\".\n-    \"\"\"\n-\n-    input_type = \"month\"\n-\n-\n-class WeekInput(Input):\n-    \"\"\"\n-    Renders an input with type \"week\".\n-    \"\"\"\n-\n-    input_type = \"week\"\n-\n-\n-class TimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"time\".\n-    \"\"\"\n-\n-    input_type = \"time\"\n-\n-\n-class DateTimeLocalInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime-local\".\n-    \"\"\"\n-\n-    input_type = \"datetime-local\"\n-\n-\n-class NumberInput(Input):\n-    \"\"\"\n-    Renders an input with type \"number\".\n-    \"\"\"\n-\n-    input_type = \"number\"\n-\n-    def __init__(self, step=None, min=None, max=None):\n-        self.step = step\n-        self.min = min\n-        self.max = max\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        if self.min is not None:\n-            kwargs.setdefault(\"min\", self.min)\n-        if self.max is not None:\n-            kwargs.setdefault(\"max\", self.max)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class RangeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"range\".\n-    \"\"\"\n-\n-    input_type = \"range\"\n-\n-    def __init__(self, step=None):\n-        self.step = step\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class ColorInput(Input):\n-    \"\"\"\n-    Renders an input with type \"color\".\n-    \"\"\"\n-\n-    input_type = \"color\"\n",
          "test_patch": "diff --git a/tests/test_fields.py b/tests/test_fields.py\nindex f36ce71..0986534 100644\n--- a/tests/test_fields.py\n+++ b/tests/test_fields.py\n@@ -119,7 +119,7 @@ class TestFlags:\n     def test_existing_values(self, flags):\n         assert flags.required is True\n         assert \"required\" in flags\n-        assert flags.optional is False\n+        assert flags.optional is None\n         assert \"optional\" not in flags\n \n     def test_assignment(self, flags):\n@@ -1373,3 +1373,111 @@ class TestHTML5Fields:\n             if field.data != item[\"data\"]:\n                 tmpl = \"Field {key} data mismatch: {field.data!r} != {data!r}\"\n                 raise AssertionError(tmpl.format(field=field, **item))\n+\n+\n+class TestFieldValidatorsTest:\n+    class F(Form):\n+        v1 = [validators.Length(min=1)]\n+        v2 = [validators.Length(min=1, max=3)]\n+        string1 = StringField(validators=v1)\n+        string2 = StringField(validators=v2)\n+        password1 = PasswordField(validators=v1)\n+        password2 = PasswordField(validators=v2)\n+        textarea1 = TextAreaField(validators=v1)\n+        textarea2 = TextAreaField(validators=v2)\n+        search1 = SearchField(validators=v1)\n+        search2 = SearchField(validators=v2)\n+\n+        v3 = [validators.NumberRange(min=1)]\n+        v4 = [validators.NumberRange(min=1, max=3)]\n+        integer1 = IntegerField(validators=v3)\n+        integer2 = IntegerField(validators=v4)\n+        integerrange1 = IntegerRangeField(validators=v3)\n+        integerrange2 = IntegerRangeField(validators=v4)\n+        decimal1 = DecimalField(validators=v3)\n+        decimal2 = DecimalField(validators=v4)\n+        decimalrange1 = DecimalRangeField(validators=v3)\n+        decimalrange2 = DecimalRangeField(validators=v4)\n+\n+    def test_minlength_maxlength(self):\n+        form = self.F()\n+        assert (\n+            form.string1()\n+            == '<input id=\"string1\" minlength=\"1\" name=\"string1\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.string2() == '<input id=\"string2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"string2\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password1() == '<input id=\"password1\" minlength=\"1\"'\n+            ' name=\"password1\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password2() == '<input id=\"password2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"password2\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.textarea1() == '<textarea id=\"textarea1\" minlength=\"1\"'\n+            ' name=\"textarea1\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.textarea2() == '<textarea id=\"textarea2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"textarea2\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.search1() == '<input id=\"search1\" minlength=\"1\"'\n+            ' name=\"search1\" type=\"search\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.search2() == '<input id=\"search2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"search2\" type=\"search\" value=\"\">'\n+        )\n+\n+    def test_min_max(self):\n+        form = self.F()\n+        assert (\n+            form.integer1()\n+            == '<input id=\"integer1\" min=\"1\" name=\"integer1\" type=\"number\" value=\"\">'\n+        )\n+        assert (\n+            form.integer2() == '<input id=\"integer2\" max=\"3\" min=\"1\"'\n+            ' name=\"integer2\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange1() == '<input id=\"integerrange1\" min=\"1\"'\n+            ' name=\"integerrange1\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange2() == '<input id=\"integerrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"integerrange2\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal1() == '<input id=\"decimal1\" min=\"1\"'\n+            ' name=\"decimal1\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal2() == '<input id=\"decimal2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimal2\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange1() == '<input id=\"decimalrange1\" min=\"1\"'\n+            ' name=\"decimalrange1\" step=\"any\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange2() == '<input id=\"decimalrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimalrange2\" step=\"any\" type=\"range\" value=\"\">'\n+        )\ndiff --git a/tests/test_validators.py b/tests/test_validators.py\nindex dd4613c..4e373ff 100644\n--- a/tests/test_validators.py\n+++ b/tests/test_validators.py\n@@ -31,7 +31,7 @@ def test_data_required(dummy_form, dummy_field):\n     validator = data_required()\n     dummy_field.data = \"foobar\"\n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n @pytest.mark.parametrize(\"bad_val\", [None, \"\", \" \", \"\\t\\t\"])\n@@ -85,7 +85,7 @@ def test_input_required(dummy_form, dummy_field):\n     dummy_field.raw_data = [\"foobar\"]\n \n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n def test_input_required_raises(dummy_form, dummy_field):\n@@ -141,7 +141,7 @@ def test_input_optional_raises(data_v, raw_data_v, dummy_form, dummy_field):\n \n     with pytest.raises(StopValidation):\n         validator(dummy_form, dummy_field)\n-        assert validator.field_flags == (\"optional\",)\n+        assert validator.field_flags == {\"optional\": True}\n \n     dummy_field.errors = [\"Invalid Integer Value\"]\n     assert len(dummy_field.errors) == 1\ndiff --git a/tests/test_widgets.py b/tests/test_widgets.py\nindex 9bedec6..a9b7cd3 100644\n--- a/tests/test_widgets.py\n+++ b/tests/test_widgets.py\n@@ -7,14 +7,14 @@ from wtforms.widgets.core import HiddenInput\n from wtforms.widgets.core import html_params\n from wtforms.widgets.core import Input\n from wtforms.widgets.core import ListWidget\n+from wtforms.widgets.core import NumberInput\n from wtforms.widgets.core import PasswordInput\n from wtforms.widgets.core import RadioInput\n+from wtforms.widgets.core import RangeInput\n from wtforms.widgets.core import Select\n from wtforms.widgets.core import TableWidget\n from wtforms.widgets.core import TextArea\n from wtforms.widgets.core import TextInput\n-from wtforms.widgets.html5 import NumberInput\n-from wtforms.widgets.html5 import RangeInput\n \n \n class TestHTMLParams:\n",
          "FAIL_TO_PASS": [
            "tests/test_fields.py::TestDefaults::test",
            "tests/test_fields.py::TestLabel::test",
            "tests/test_fields.py::TestLabel::test_auto_label",
            "tests/test_fields.py::TestLabel::test_override_for",
            "tests/test_fields.py::TestLabel::test_escaped_label_text",
            "tests/test_fields.py::TestFlags::test_existing_values",
            "tests/test_fields.py::TestFlags::test_assignment",
            "tests/test_fields.py::TestFlags::test_unset",
            "tests/test_fields.py::TestFlags::test_repr",
            "tests/test_fields.py::TestFlags::test_underscore_property",
            "tests/test_fields.py::TestUnsetValue::test",
            "tests/test_fields.py::TestFilters::test_working",
            "tests/test_fields.py::TestFilters::test_failure",
            "tests/test_fields.py::TestFilters::test_extra",
            "tests/test_fields.py::TestFilters::test_inline",
            "tests/test_fields.py::TestField::test_unbound_field",
            "tests/test_fields.py::TestField::test_htmlstring",
            "tests/test_fields.py::TestField::test_str_coerce",
            "tests/test_fields.py::TestField::test_unicode_coerce",
            "tests/test_fields.py::TestField::test_process_formdata",
            "tests/test_fields.py::TestField::test_meta_attribute",
            "tests/test_fields.py::TestField::test_render_kw",
            "tests/test_fields.py::TestField::test_render_special",
            "tests/test_fields.py::TestField::test_select_field_copies_choices",
            "tests/test_fields.py::TestField::test_required_flag",
            "tests/test_fields.py::TestField::test_check_validators",
            "tests/test_fields.py::TestField::test_custom_name",
            "tests/test_fields.py::TestPrePostValidation::test_pre_stop",
            "tests/test_fields.py::TestPrePostValidation::test_post",
            "tests/test_fields.py::TestSelectField::test_defaults",
            "tests/test_fields.py::TestSelectField::test_with_data",
            "tests/test_fields.py::TestSelectField::test_value_coercion",
            "tests/test_fields.py::TestSelectField::test_iterable_options",
            "tests/test_fields.py::TestSelectField::test_default_coerce",
            "tests/test_fields.py::TestSelectField::test_validate_choices",
            "tests/test_fields.py::TestSelectField::test_validate_choices_when_empty",
            "tests/test_fields.py::TestSelectField::test_validate_choices_when_none",
            "tests/test_fields.py::TestSelectField::test_dont_validate_choices",
            "tests/test_fields.py::TestSelectField::test_choice_shortcut",
            "tests/test_fields.py::TestSelectField::test_empty_choice[choices0]",
            "tests/test_fields.py::TestSelectField::test_empty_choice[None]",
            "tests/test_fields.py::TestSelectField::test_callable_choices",
            "tests/test_fields.py::TestSelectMultipleField::test_defaults",
            "tests/test_fields.py::TestSelectMultipleField::test_with_data",
            "tests/test_fields.py::TestSelectMultipleField::test_coerce_fail",
            "tests/test_fields.py::TestSelectMultipleField::test_callable_choices",
            "tests/test_fields.py::TestSelectMultipleField::test_choice_shortcut",
            "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[choices0]",
            "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[None]",
            "tests/test_fields.py::TestRadioField::test",
            "tests/test_fields.py::TestRadioField::test_text_coercion",
            "tests/test_fields.py::TestRadioField::test_callable_choices",
            "tests/test_fields.py::TestStringField::test",
            "tests/test_fields.py::TestHiddenField::test",
            "tests/test_fields.py::TestTextAreaField::test",
            "tests/test_fields.py::TestPasswordField::test",
            "tests/test_fields.py::TestFileField::test_file_field",
            "tests/test_fields.py::TestFileField::test_multiple_file_field",
            "tests/test_fields.py::TestFileField::test_file_field_without_file_input",
            "tests/test_fields.py::TestIntegerField::test",
            "tests/test_fields.py::TestDecimalField::test",
            "tests/test_fields.py::TestDecimalField::test_quantize",
            "tests/test_fields.py::TestFloatField::test",
            "tests/test_fields.py::TestBooleanField::test_defaults",
            "tests/test_fields.py::TestBooleanField::test_rendering",
            "tests/test_fields.py::TestBooleanField::test_with_postdata",
            "tests/test_fields.py::TestBooleanField::test_with_model_data",
            "tests/test_fields.py::TestBooleanField::test_with_postdata_and_model",
            "tests/test_fields.py::TestDateField::test_basic",
            "tests/test_fields.py::TestDateField::test_failure",
            "tests/test_fields.py::TestMonthField::test_basic",
            "tests/test_fields.py::TestMonthField::test_failure",
            "tests/test_fields.py::TestTimeField::test_basic",
            "tests/test_fields.py::TestDateTimeField::test_basic",
            "tests/test_fields.py::TestDateTimeField::test_microseconds",
            "tests/test_fields.py::TestSubmitField::test",
            "tests/test_fields.py::TestFormField::test_formdata",
            "tests/test_fields.py::TestFormField::test_iteration",
            "tests/test_fields.py::TestFormField::test_with_obj",
            "tests/test_fields.py::TestFormField::test_widget",
            "tests/test_fields.py::TestFormField::test_separator",
            "tests/test_fields.py::TestFormField::test_no_validators_or_filters",
            "tests/test_fields.py::TestFormField::test_populate_missing_obj",
            "tests/test_fields.py::TestFieldList::test_form",
            "tests/test_fields.py::TestFieldList::test_enclosed_subform",
            "tests/test_fields.py::TestFieldList::test_enclosed_subform_custom_name",
            "tests/test_fields.py::TestFieldList::test_entry_management",
            "tests/test_fields.py::TestFieldList::test_min_max_entries",
            "tests/test_fields.py::TestFieldList::test_validators",
            "tests/test_fields.py::TestFieldList::test_no_filters",
            "tests/test_fields.py::TestFieldList::test_process_prefilled",
            "tests/test_fields.py::TestFieldList::test_errors",
            "tests/test_fields.py::TestCustomFieldQuirks::test_processing_failure",
            "tests/test_fields.py::TestCustomFieldQuirks::test_default_impls",
            "tests/test_fields.py::TestHTML5Fields::test_simple",
            "tests/test_fields.py::TestFieldValidatorsTest::test_minlength_maxlength",
            "tests/test_fields.py::TestFieldValidatorsTest::test_min_max",
            "tests/test_validators.py::test_data_required",
            "tests/test_validators.py::test_data_required_raises[None]",
            "tests/test_validators.py::test_data_required_raises[]",
            "tests/test_validators.py::test_data_required_raises[",
            "tests/test_validators.py::test_data_required_raises[\\t\\t]",
            "tests/test_validators.py::test_data_required_clobber",
            "tests/test_validators.py::test_data_required_messages[validator0-This",
            "tests/test_validators.py::test_data_required_messages[validator1-foo]",
            "tests/test_validators.py::test_input_required",
            "tests/test_validators.py::test_input_required_raises",
            "tests/test_validators.py::test_input_required_error_message[validator0-This",
            "tests/test_validators.py::test_input_required_error_message[validator1-foo]",
            "tests/test_validators.py::test_input_optional_passes",
            "tests/test_validators.py::test_input_optional_raises[-]",
            "tests/test_validators.py::test_input_optional_raises[",
            "tests/test_validators.py::test_input_optional_raises[\\t-\\t]",
            "tests/test_validators.py::test_ip4address_passes[147.230.23.25]",
            "tests/test_validators.py::test_ip4address_passes[147.230.23.0]",
            "tests/test_validators.py::test_ip4address_passes[127.0.0.1]",
            "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::1111]",
            "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::]",
            "tests/test_validators.py::test_good_ip6address_passes[::1]",
            "tests/test_validators.py::test_good_ip6address_passes[dead:beef:0:0:0:0:42:1]",
            "tests/test_validators.py::test_good_ip6address_passes[abcd:ef::42:1]",
            "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::1111]",
            "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::]",
            "tests/test_validators.py::test_bad_ip6address_raises[abc.0.0.1]",
            "tests/test_validators.py::test_bad_ip6address_raises[abcd:1234::123::1]",
            "tests/test_validators.py::test_bad_ip6address_raises[1:2:3:4:5:6:7:8:9]",
            "tests/test_validators.py::test_bad_ip6address_raises[abcd::1ffff]",
            "tests/test_validators.py::test_bad_ip4address_raises[147.230.1000.25]",
            "tests/test_validators.py::test_bad_ip4address_raises[2001:718::::]",
            "tests/test_validators.py::test_bad_ip4address_raises[abc.0.0.1]",
            "tests/test_validators.py::test_bad_ip4address_raises[1278.0.0.1]",
            "tests/test_validators.py::test_bad_ip4address_raises[127.0.0.abc]",
            "tests/test_validators.py::test_bad_ip4address_raises[900.200.100.75]",
            "tests/test_validators.py::test_valid_email_passes[foo@bar.dk]",
            "tests/test_validators.py::test_valid_email_passes[123@bar.dk]",
            "tests/test_validators.py::test_valid_email_passes[foo@456.dk]",
            "tests/test_validators.py::test_valid_email_passes[foo@bar456.info]",
            "tests/test_validators.py::test_valid_email_passes[foo@b\\xfccher.\\u4e2d\\u56fd]",
            "tests/test_validators.py::test_invalid_email_raises[None]",
            "tests/test_validators.py::test_invalid_email_raises[]",
            "tests/test_validators.py::test_invalid_email_raises[",
            "tests/test_validators.py::test_invalid_email_raises[foo]",
            "tests/test_validators.py::test_invalid_email_raises[bar.dk]",
            "tests/test_validators.py::test_invalid_email_raises[foo@]",
            "tests/test_validators.py::test_invalid_email_raises[@bar.dk]",
            "tests/test_validators.py::test_invalid_email_raises[foo@bar]",
            "tests/test_validators.py::test_invalid_email_raises[foo@bar.ab12]",
            "tests/test_validators.py::test_invalid_email_raises[foo@.bar.ab]",
            "tests/test_validators.py::test_invalid_email_raises[foo.@bar.co]",
            "tests/test_validators.py::test_invalid_email_raises[foo@foo@bar.co]",
            "tests/test_validators.py::test_invalid_email_raises[fo",
            "tests/test_validators.py::test_invalid_email_raises_granular_message[foo@]",
            "tests/test_validators.py::test_ip_address_raises_on_bad_init",
            "tests/test_validators.py::test_valid_mac_passes[01:23:45:67:ab:CD]",
            "tests/test_validators.py::test_bad_mac_raises[00:00:00:00:00]",
            "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:]",
            "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:gh]",
            "tests/test_validators.py::test_bad_mac_raises[123:23:45:67:89:00]",
            "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f-0deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f0deb43e992a14775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-0deb-43e9-92a1-4775189ec9f]",
            "tests/test_validators.py::test_bad_uuid_raises[gbc1c94f-0deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f",
            "tests/test_validators.py::test_required_passes",
            "tests/test_validators.py::test_required_stops_validation",
            "tests/test_validators.py::test_equal_to_passes",
            "tests/test_validators.py::test_equal_to_raises[test-invalid_field_name]",
            "tests/test_validators.py::test_equal_to_raises[bad_value-foo]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo-bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo--bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo_bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk?query=param]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.museum/foobar]",
            "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1/foobar]",
            "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1:9000/fake]",
            "tests/test_validators.py::test_valid_url_passes[http://\\u0645\\u062b\\u0627\\u0644.\\u0625\\u062e\\u062a\\u0628\\u0627\\u0631/foo.com]",
            "tests/test_validators.py::test_valid_url_passes[http://\\u0909\\u0926\\u093e\\u0939\\u0930\\u0923.\\u092a\\u0930\\u0940\\u0915\\u094d\\u0937\\u093e/]",
            "tests/test_validators.py::test_valid_url_passes[http://\\uc2e4\\ub840.\\ud14c\\uc2a4\\ud2b8]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://localhost/foobar]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:5000?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar/path?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:1234/path?query=param&foo=faa]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar]",
            "tests/test_validators.py::test_bad_url_raises[http://-foobar.dk/]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar-.dk/]",
            "tests/test_validators.py::test_bad_url_raises[foobar.dk]",
            "tests/test_validators.py::test_bad_url_raises[http://127.0.0/asdf]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar.d]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar.12]",
            "tests/test_validators.py::test_bad_url_raises[http://localhost:abc/a]",
            "tests/test_validators.py::test_anyof_passes[b-test_list0]",
            "tests/test_validators.py::test_anyof_passes[2-test_list1]",
            "tests/test_validators.py::test_anyof_raisses[d-test_list0]",
            "tests/test_validators.py::test_anyof_raisses[6-test_list1]",
            "tests/test_validators.py::test_any_of_values_formatter",
            "tests/test_validators.py::test_none_of_passes",
            "tests/test_validators.py::test_none_of_raises",
            "tests/test_validators.py::test_correct_length_passes[2-6]",
            "tests/test_validators.py::test_correct_length_passes[6--1]",
            "tests/test_validators.py::test_correct_length_passes[-1-6]",
            "tests/test_validators.py::test_correct_length_passes[6-6]",
            "tests/test_validators.py::test_bad_length_raises[7--1]",
            "tests/test_validators.py::test_bad_length_raises[-1-5]",
            "tests/test_validators.py::test_bad_length_init_raises[-1--1]",
            "tests/test_validators.py::test_bad_length_init_raises[5-2]",
            "tests/test_validators.py::test_length_messages[validator0-2",
            "tests/test_validators.py::test_length_messages[validator1-at",
            "tests/test_validators.py::test_length_messages[validator2-longer",
            "tests/test_validators.py::test_length_messages[validator3-between",
            "tests/test_validators.py::test_length_messages[validator4-exactly",
            "tests/test_validators.py::test_number_range_passes[5-10-7]",
            "tests/test_validators.py::test_number_range_passes[5-None-7]",
            "tests/test_validators.py::test_number_range_passes[None-100-70]",
            "tests/test_validators.py::test_number_range_raises[5-10-None]",
            "tests/test_validators.py::test_number_range_raises[5-10-0]",
            "tests/test_validators.py::test_number_range_raises[5-10-12]",
            "tests/test_validators.py::test_number_range_raises[5-10--5]",
            "tests/test_validators.py::test_number_range_raises[5-None-4]",
            "tests/test_validators.py::test_number_range_raises[None-100-500]",
            "tests/test_validators.py::test_number_range_nan[nan]",
            "tests/test_validators.py::test_number_range_nan[nan1]",
            "tests/test_validators.py::test_lazy_proxy_raises",
            "tests/test_validators.py::test_lazy_proxy_fixture",
            "tests/test_validators.py::test_regex_passes[^a-None-abcd-a0]",
            "tests/test_validators.py::test_regex_passes[^a-re.IGNORECASE-ABcd-A]",
            "tests/test_validators.py::test_regex_passes[^a-None-abcd-a1]",
            "tests/test_validators.py::test_regex_passes[^a-None-ABcd-A]",
            "tests/test_validators.py::test_regex_raises[^a-None-ABC]",
            "tests/test_validators.py::test_regex_raises[^a-re.IGNORECASE-foo]",
            "tests/test_validators.py::test_regex_raises[^a-None-None0]",
            "tests/test_validators.py::test_regex_raises[^a-None-foo]",
            "tests/test_validators.py::test_regex_raises[^a-None-None1]",
            "tests/test_validators.py::test_regexp_message",
            "tests/test_validators.py::test_raise_exceptions[IndexError]",
            "tests/test_validators.py::test_raise_exceptions[ZeroDivisionError]",
            "tests/test_validators.py::test_raise_exceptions[ValueError]",
            "tests/test_widgets.py::TestHTMLParams::test_basic",
            "tests/test_widgets.py::TestHTMLParams::test_data_prefix",
            "tests/test_widgets.py::TestHTMLParams::test_aria_prefix",
            "tests/test_widgets.py::TestHTMLParams::test_quoting",
            "tests/test_widgets.py::TestListWidget::test_listwidget",
            "tests/test_widgets.py::TestTableWidget::test_tablewidget",
            "tests/test_widgets.py::TestBasicWidgets::test_input_type",
            "tests/test_widgets.py::TestBasicWidgets::test_html_marking",
            "tests/test_widgets.py::TestBasicWidgets::test_text_input",
            "tests/test_widgets.py::TestBasicWidgets::test_password_input",
            "tests/test_widgets.py::TestBasicWidgets::test_hidden_input",
            "tests/test_widgets.py::TestBasicWidgets::test_checkbox_input",
            "tests/test_widgets.py::TestBasicWidgets::test_radio_input",
            "tests/test_widgets.py::TestBasicWidgets::test_textarea",
            "tests/test_widgets.py::TestBasicWidgets::test_file",
            "tests/test_widgets.py::TestSelect::test_select",
            "tests/test_widgets.py::TestSelect::test_render_option",
            "tests/test_widgets.py::TestHTML5::test_number",
            "tests/test_widgets.py::TestHTML5::test_range"
          ],
          "PASS_TO_PASS": [],
          "install_config": {
            "base_image_name": "python_base_310",
            "docker_specs": {
              "_variant": null,
              "bazel_version": null,
              "bun_version": null,
              "cargo_version": null,
              "deno_version": null,
              "docker_version": null,
              "erlang_version": null,
              "gcc_version": null,
              "go_version": null,
              "helm_version": null,
              "java_version": null,
              "jdk_version": null,
              "llvm_version": null,
              "lua_version": null,
              "luajit_version": null,
              "neovim_version": null,
              "node_version": null,
              "npm_version": null,
              "nvim_version": null,
              "pnpm_version": null,
              "python_image": "python_base_310",
              "python_version": null,
              "redis_version": null,
              "ruby_version": null,
              "rust_version": null,
              "rustc_version": null,
              "solana_version": null,
              "sqlite_version": null
            },
            "install": [
              "pip install -q -e .[email] pytest babel"
            ],
            "log_parser": "parse_log_pytest",
            "test_cmd": "pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning tests/test_fields.py tests/test_validators.py tests/test_widgets.py"
          },
          "image_name": "docker.io/swerebenchv2/wtforms-wtforms:614-848d28d",
          "source_has_task_specific_rubric": false
        },
        "origin": "drafted",
        "criterion_count": 3
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Pinned issue, repair patch, test patch and environment configuration are available."
        ],
        "risks": [
          "The source interface summary includes many details beyond the short issue; visible task context must include that contract.",
          "Repository/image was not reconstructed or executed for this gallery."
        ],
        "must_hold": [
          "Default widgets reflect the requested HTML5 behavior.",
          "Field/validator flags and public interfaces remain coherent."
        ],
        "may_vary": [
          "Implementation factoring across widget helpers.",
          "Equivalent code organization and generated HTML where semantics and tested contract agree."
        ],
        "proposed_changes": [
          "Convert behavior and test obligations into criteria; accept different correct patches.",
          "Run failing/passing tests plus targeted uncovered cases before treating any probe as a verified repair."
        ]
      },
      "probes": [
        {
          "label": "Central widget helper sketch",
          "kind": "valid_alternative",
          "response": "Centralize allowed validation-attribute propagation in the common input rendering helper and have concrete widgets declare their supported attributes.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "A potential implementation organization, pending test execution.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Concrete widget delegation sketch",
          "kind": "valid_alternative",
          "response": "Have concrete widgets delegate through a shared attribute builder while preserving the documented field/validator flag interactions.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "An alternate internal arrangement can preserve the same public behavior.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "String replacement only",
          "kind": "invalid_control",
          "response": "Replace type=\"text\" with type=\"email\" in every rendered input.",
          "is_full_response": false,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "unexecuted_sketch",
          "why": "Destroys the semantics of non-email fields and does not implement the interface contract.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "base_commit": "848d28d67e45cda7a06c4c8ed2768e6a8cb1c016",
        "created_at": "2020-05-31 15:27:12",
        "image_name": "docker.io/swerebenchv2/wtforms-wtforms:614-848d28d",
        "instance_id": "wtforms__wtforms-614",
        "interface": "Method: Flags.__getattr__(self, name)  \nLocation: wtforms/fields/core.py → class Flags  \nInputs: *name* – attribute name requested on a Flags instance.  \nOutputs: Returns the attribute’s value if it exists; otherwise returns **None** (previously returned False).  \nDescription: Provides a unified flags object where missing flags evaluate to None, enabling flag‑based widget rendering.\n\nMethod: Input.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class Input  \nInputs: *field* – a WTForms Field instance; **kwargs – optional HTML attributes passed at render time.  \nOutputs: A Markup string for an `<input …>` element; automatically injects any flag attributes whose names appear in the widget’s *validation_attrs* list and are set on *field.flags*.  \nDescription: Renders HTML5‑compatible input elements, pulling validation‑related attributes (e.g., required, minlength, maxlength, min, max, step) from the field’s flags.\n\nMethod: TextInput.__call__(self, field, **kwargs) *(inherited, uses Input.__call__)*  \nLocation: wtforms/widgets/core.py → class TextInput  \nInputs: Same as Input.__call__.  \nOutputs: `<input type=\"text\" …>` markup with added validation attributes.  \nDescription: Text input widget that now supports required, maxlength, minlength, and pattern attributes derived from field flags.\n\nMethod: PasswordInput.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class PasswordInput  \nInputs: Same as Input.__call__.  \nOutputs: `<input type=\"password\" …>` markup with validation attributes.  \nDescription: Password input widget supporting the same validation flags as TextInput.\n\nMethod: TextArea.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class TextArea  \nInputs: *field*; **kwargs.  \nOutputs: `<textarea …>` markup with required, maxlength, minlength attributes taken from flags.  \nDescription: Textarea widget now propagates flag‑based validation attributes.\n\nMethod: Select.__call__(self, field, **kwargs)  \nLocation: wtforms/widgets/core.py → class Select  \nInputs: *field*; **kwargs.  \nOutputs: `<select …>` markup with required attribute from flags.  \nDescription: Select widget now adds a required attribute when the field’s flags indicate it.\n\nMethod: SearchField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class SearchField (inherits StringField)  \nInputs: *label*, *validators*, additional field kwargs.  \nOutputs: Instance of a field rendering `<input type=\"search\">`.  \nDescription: New convenience field for HTML `search` input type, using the SearchInput widget.\n\nMethod: TelField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class TelField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"tel\">`.  \nDescription: New field for telephone inputs.\n\nMethod: URLField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class URLField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"url\">`.  \nDescription: New field for URL inputs.\n\nMethod: EmailField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class EmailField (inherits StringField)  \nInputs: Same as SearchField.  \nOutputs: Instance rendering `<input type=\"email\">`.  \nDescription: New field for email inputs.\n\nMethod: DateTimeLocalField.__init__(self, label=None, validators=None, format=\"%Y-%m-%d %H:%M:%S\", **kwargs)  \nLocation: wtforms/fields/core.py → class DateTimeLocalField (inherits DateTimeField)  \nInputs: *label*, *validators*, *format*, additional kwargs.  \nOutputs: Instance rendering `<input type=\"datetime-local\">`.  \nDescription: New field for HTML5 datetime‑local inputs.\n\nMethod: IntegerRangeField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class IntegerRangeField (inherits IntegerField)  \nInputs: Same as IntegerField.  \nOutputs: Instance rendering `<input type=\"range\">`.  \nDescription: New field for numeric range inputs, using the RangeInput widget.\n\nMethod: DecimalRangeField.__init__(self, label=None, validators=None, **kwargs)  \nLocation: wtforms/fields/core.py → class DecimalRangeField (inherits DecimalField)  \nInputs: Same as DecimalField.  \nOutputs: Instance rendering `<input type=\"range\">` with step=\"any\".  \nDescription: New field for decimal range inputs.\n\nMethod: Length.__init__(self, min=-1, max=-1, message=None)  \nLocation: wtforms/validators.py → class Length  \nInputs: *min* (int, default –1), *max* (int, default –1), *message* (optional).  \nOutputs: Validator instance with *field_flags* dict containing “minlength” and/or “maxlength” when limits are set.  \nDescription: Length validator now provides flag information for widget rendering.\n\nMethod: NumberRange.__init__(self, min=None, max=None, message=None)  \nLocation: wtforms/validators.py → class NumberRange  \nInputs: *min* (numeric or None), *max* (numeric or None), *message*.  \nOutputs: Validator instance with *field_flags* dict containing “min” and/or “max”.  \nDescription: NumberRange validator now supplies min/max flags for HTML5 widgets.\n\nMethod: Optional.__init__(self, strip_whitespace=True)  \nLocation: wtforms/validators.py → class Optional  \nInputs: *strip_whitespace* (bool).  \nOutputs: Validator instance with *field_flags* = {\"optional\": True}.  \nDescription: Optional validator now sets an explicit “optional” flag for widgets.\n\nMethod: DataRequired.__init__(self, message=None)  \nLocation: wtforms/validators.py → class DataRequired  \nInputs: *message* (optional).  \nOutputs: Validator instance with *field_flags* = {\"required\": True}.  \nDescription: DataRequired now provides a required flag for widget rendering.\n\nMethod: InputRequired.__init__(self, message=None)  \nLocation: wtforms/validators.py → class InputRequired  \nInputs: *message* (optional).  \nOutputs: Validator instance with *field_flags* = {\"required\": True}.  \nDescription: InputRequired now provides the required flag similarly to DataRequired.\n\nFunction: html_params(**kwargs) *(unchanged signature, but now used by widgets to include flag attributes)*  \nLocation: wtforms/widgets/core.py → helper function.  \nInputs: Arbitrary HTML attribute key/value pairs.  \nOutputs: Properly escaped attribute string.  \nDescription: Generates the attribute list for widget rendering, now receives additional validation attributes from flags.\n\n",
        "language": "python",
        "license": "BSD-3-Clause",
        "patch": "diff --git a/CHANGES.rst b/CHANGES.rst\nindex 50f43a2..a199f46 100644\n--- a/CHANGES.rst\n+++ b/CHANGES.rst\n@@ -34,7 +34,7 @@ Unreleased\n -   Render attribute names like ``for_`` and ``class_`` are normalized\n     consistently so later values override those specified earlier.\n     :issue:`449`, :pr:`596`\n-\n+-   Flags can take non-boolean values. :issue:`406` :pr:`467`\n \n Version 2.3.1\n -------------\ndiff --git a/docs/fields.rst b/docs/fields.rst\nindex ed047ce..0bd3e32 100644\n--- a/docs/fields.rst\n+++ b/docs/fields.rst\n@@ -182,10 +182,10 @@ The Field base class\n \n     .. attribute:: flags\n \n-        An object containing boolean flags set either by the field itself, or\n+        An object containing flags set either by the field itself, or\n         by validators on the field. For example, the built-in\n         :class:`~wtforms.validators.InputRequired` validator sets the `required` flag.\n-        An unset flag will result in :const:`False`.\n+        An unset flag will result in :const:`None`.\n \n         .. code-block:: html+django\n \n@@ -220,10 +220,14 @@ refer to a single input from the form.\n \n .. autoclass:: DateTimeField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n-    For better date/time fields, see the :mod:`dateutil extension <wtforms.ext.dateutil.fields>`\n+.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n .. autoclass:: DecimalField(default field arguments, places=2, rounding=None, use_locale=False, number_format=None)\n \n+.. autoclass:: DecimalRangeField(default field arguments)\n+\n+.. autoclass:: EmailField(default field arguments)\n+\n .. autoclass:: FileField(default field arguments)\n \n     Example usage::\n@@ -252,6 +256,8 @@ refer to a single input from the form.\n \n .. autoclass:: IntegerField(default field arguments)\n \n+.. autoclass:: IntegerRangeField(default field arguments)\n+\n .. autoclass:: RadioField(default field arguments, choices=[], coerce=str)\n \n     .. code-block:: jinja\n@@ -323,6 +329,8 @@ refer to a single input from the form.\n     a list of fields each representing an option. The rendering of this can be\n     further controlled by specifying `option_widget=`.\n \n+.. autoclass:: SearchField(default field arguments)\n+\n .. autoclass:: SelectMultipleField(default field arguments, choices=[], coerce=str, option_widget=None)\n \n    The data on the SelectMultipleField is stored as a list of objects, each of\n@@ -338,6 +346,13 @@ refer to a single input from the form.\n \n         {{ form.username(size=30, maxlength=50) }}\n \n+.. autoclass:: TelField(default field arguments)\n+\n+.. autoclass:: TimeField(default field arguments, format='%H:%M')\n+\n+.. autoclass:: URLField(default field arguments)\n+\n+\n Convenience Fields\n ------------------\n \n@@ -559,40 +574,3 @@ Additional Helper Classes\n     .. attribute:: text\n \n         The original label text passed to the field's constructor.\n-\n-HTML5 Fields\n-------------\n-\n-In addition to basic HTML fields, WTForms also supplies fields for the HTML5\n-standard. These fields can be accessed under the :mod:`wtforms.fields.html5` namespace.\n-In reality, these fields are just convenience fields that extend basic fields\n-and implement HTML5 specific widgets. These widgets are located in the :mod:`wtforms.widgets.html5`\n-namespace and can be overridden or modified just like any other widget.\n-\n-.. module:: wtforms.fields.html5\n-\n-.. autoclass:: SearchField(default field arguments)\n-\n-.. autoclass:: TelField(default field arguments)\n-\n-.. autoclass:: URLField(default field arguments)\n-\n-.. autoclass:: EmailField(default field arguments)\n-\n-.. autoclass:: DateTimeField(default field arguments,  format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: DateField(default field arguments, format='%Y-%m-%d')\n-\n-.. autoclass:: MonthField(default field arguments, format='%Y-%m')\n-\n-.. autoclass:: TimeField(default field arguments, format='%H:%M')\n-\n-.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: IntegerField(default field arguments)\n-\n-.. autoclass:: DecimalField(default field arguments)\n-\n-.. autoclass:: IntegerRangeField(default field arguments)\n-\n-.. autoclass:: DecimalRangeField(default field arguments)\ndiff --git a/docs/validators.rst b/docs/validators.rst\nindex cdb9627..818c018 100644\n--- a/docs/validators.rst\n+++ b/docs/validators.rst\n@@ -177,22 +177,21 @@ which will then be available on the :attr:`field's flags object\n \n To specify flags on your validator, set the ``field_flags`` attribute on your\n validator. When the Field is constructed, the flags with the same name will be\n-set to True on your field. For example, let's imagine a validator that\n+set on your field. For example, let's imagine a validator that\n validates that input is valid BBCode. We can set a flag on the field then to\n signify that the field accepts BBCode::\n \n     # class implementation\n     class ValidBBCode(object):\n-        field_flags = ('accepts_bbcode', )\n-\n-        pass # validator implementation here\n+        def __init__(self):\n+            self.field_flags = {'accepts_bbcode': True}\n \n     # factory implementation\n     def valid_bbcode():\n         def _valid_bbcode(form, field):\n             pass # validator implementation here\n \n-        _valid_bbcode.field_flags = ('accepts_bbcode', )\n+        _valid_bbcode.field_flags = {'accepts_bbcode': True}\n         return _valid_bbcode\n \n Then we can check it in our template, so we can then place a note to the user:\n@@ -206,7 +205,10 @@ Then we can check it in our template, so we can then place a note to the user:\n \n Some considerations on using flags:\n \n-* Flags can only set boolean values, and another validator cannot unset them.\n-* If multiple fields set the same flag, its value is still True.\n+* Boolean flags will set HTML valueless attributes (e.g. `{required: True}`\n+  will give `<input type=\"text\" required>`). Other flag types will set regular\n+  HTML attributes (e.g. `{maxlength: 8}` will give `<input type=\"text\" maxlength=\"8\">`).\n+* If multiple validators set the same flag, the flag will have the value set\n+  by the first validator.\n * Flags are set from validators only in :meth:`Field.__init__`, so inline\n   validators and extra passed-in validators cannot set them.\ndiff --git a/docs/widgets.rst b/docs/widgets.rst\nindex 81b96c0..5c4f009 100644\n--- a/docs/widgets.rst\n+++ b/docs/widgets.rst\n@@ -14,36 +14,30 @@ recognize as not needing to be auto-escaped.\n Built-in widgets\n ----------------\n \n-.. autoclass:: wtforms.widgets.ListWidget\n-.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.ColorInput\n+.. autoclass:: wtforms.widgets.CheckboxInput\n+.. autoclass:: wtforms.widgets.DateTimeInput\n+.. autoclass:: wtforms.widgets.DateTimeLocalInput\n+.. autoclass:: wtforms.widgets.DateInput\n+.. autoclass:: wtforms.widgets.EmailInput\n+.. autoclass:: wtforms.widgets.FileInput\n+.. autoclass:: wtforms.widgets.HiddenInput\n .. autoclass:: wtforms.widgets.Input\n-.. autoclass:: wtforms.widgets.TextInput()\n+.. autoclass:: wtforms.widgets.ListWidget\n+.. autoclass:: wtforms.widgets.MonthInput\n+.. autoclass:: wtforms.widgets.NumberInput\n .. autoclass:: wtforms.widgets.PasswordInput\n-.. autoclass:: wtforms.widgets.HiddenInput()\n-.. autoclass:: wtforms.widgets.CheckboxInput()\n-.. autoclass:: wtforms.widgets.FileInput()\n-.. autoclass:: wtforms.widgets.SubmitInput()\n-.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.RangeInput\n+.. autoclass:: wtforms.widgets.SubmitInput\n+.. autoclass:: wtforms.widgets.SearchInput\n .. autoclass:: wtforms.widgets.Select\n-\n-HTML5 widgets\n--------------\n-\n-.. module:: wtforms.widgets.html5\n-\n-.. autoclass:: wtforms.widgets.html5.ColorInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeLocalInput\n-.. autoclass:: wtforms.widgets.html5.DateInput\n-.. autoclass:: wtforms.widgets.html5.EmailInput\n-.. autoclass:: wtforms.widgets.html5.MonthInput\n-.. autoclass:: wtforms.widgets.html5.NumberInput\n-.. autoclass:: wtforms.widgets.html5.RangeInput\n-.. autoclass:: wtforms.widgets.html5.SearchInput\n-.. autoclass:: wtforms.widgets.html5.TelInput\n-.. autoclass:: wtforms.widgets.html5.TimeInput\n-.. autoclass:: wtforms.widgets.html5.URLInput\n-.. autoclass:: wtforms.widgets.html5.WeekInput\n+.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.TelInput\n+.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.TextInput\n+.. autoclass:: wtforms.widgets.TimeInput\n+.. autoclass:: wtforms.widgets.URLInput\n+.. autoclass:: wtforms.widgets.WeekInput\n \n Widget-Building Utilities\n -------------------------\ndiff --git a/src/wtforms/fields/__init__.py b/src/wtforms/fields/__init__.py\nindex 1f770d3..e429631 100644\n--- a/src/wtforms/fields/__init__.py\n+++ b/src/wtforms/fields/__init__.py\n@@ -3,6 +3,5 @@ from wtforms.fields.core import Field\n from wtforms.fields.core import Flags\n from wtforms.fields.core import Label\n from wtforms.fields.core import SelectFieldBase\n-from wtforms.fields.html5 import *\n from wtforms.fields.simple import *\n from wtforms.utils import unset_value as _unset_value\ndiff --git a/src/wtforms/fields/core.py b/src/wtforms/fields/core.py\nindex d835092..8724356 100644\n--- a/src/wtforms/fields/core.py\n+++ b/src/wtforms/fields/core.py\n@@ -2,6 +2,7 @@ import datetime\n import decimal\n import inspect\n import itertools\n+import warnings\n \n from markupsafe import escape\n from markupsafe import Markup\n@@ -15,18 +16,25 @@ from wtforms.validators import ValidationError\n __all__ = (\n     \"BooleanField\",\n     \"DecimalField\",\n+    \"DecimalRangeField\",\n     \"DateField\",\n     \"DateTimeField\",\n+    \"DateTimeLocalField\",\n+    \"EmailField\",\n     \"FieldList\",\n     \"FloatField\",\n     \"FormField\",\n     \"IntegerField\",\n+    \"IntegerRangeField\",\n     \"RadioField\",\n+    \"SearchField\",\n     \"SelectField\",\n     \"SelectMultipleField\",\n     \"StringField\",\n+    \"TelField\",\n     \"TimeField\",\n     \"MonthField\",\n+    \"URLField\",\n )\n \n \n@@ -143,9 +151,21 @@ class Field:\n             self.widget = widget\n \n         for v in itertools.chain(self.validators, [self.widget]):\n-            flags = getattr(v, \"field_flags\", ())\n-            for f in flags:\n-                setattr(self.flags, f, True)\n+            flags = getattr(v, \"field_flags\", {})\n+\n+            # check for legacy format, remove eventually\n+            if isinstance(flags, tuple):\n+                warnings.warn(\n+                    \"Flags should be stored in dicts and not in tuples. \"\n+                    \"The next version of WTForms will abandon support \"\n+                    \"for flags in tuples.\",\n+                    DeprecationWarning,\n+                    stacklevel=2,\n+                )\n+                flags = {flag_name: True for flag_name in flags}\n+\n+            for k, v in flags.items():\n+                setattr(self.flags, k, v)\n \n     def __str__(self):\n         \"\"\"\n@@ -408,15 +428,15 @@ class UnboundField:\n \n class Flags:\n     \"\"\"\n-    Holds a set of boolean flags as attributes.\n+    Holds a set of flags as attributes.\n \n-    Accessing a non-existing attribute returns False for its value.\n+    Accessing a non-existing attribute returns None for its value.\n     \"\"\"\n \n     def __getattr__(self, name):\n         if name.startswith(\"_\"):\n             return super().__getattr__(name)\n-        return False\n+        return None\n \n     def __contains__(self, name):\n         return getattr(self, name)\n@@ -670,7 +690,7 @@ class IntegerField(Field):\n     is ignored and will not be accepted as a value.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput()\n \n     def __init__(self, label=None, validators=None, **kwargs):\n         super().__init__(label, validators, **kwargs)\n@@ -721,7 +741,7 @@ class DecimalField(LocaleAwareNumberField):\n         format for the locale.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput(step=\"any\")\n \n     def __init__(\n         self, label=None, validators=None, places=unset_value, rounding=None, **kwargs\n@@ -842,7 +862,7 @@ class DateTimeField(Field):\n     A text field which stores a `datetime.datetime` matching a format.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.DateTimeInput()\n \n     def __init__(\n         self, label=None, validators=None, format=\"%Y-%m-%d %H:%M:%S\", **kwargs\n@@ -871,6 +891,8 @@ class DateField(DateTimeField):\n     Same as DateTimeField, except stores a `datetime.date`.\n     \"\"\"\n \n+    widget = widgets.DateInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m-%d\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -889,6 +911,8 @@ class TimeField(DateTimeField):\n     Same as DateTimeField, except stores a `time`.\n     \"\"\"\n \n+    widget = widgets.TimeInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%H:%M\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -908,6 +932,8 @@ class MonthField(DateField):\n     with `day = 1`.\n     \"\"\"\n \n+    widget = widgets.MonthInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -1189,3 +1215,59 @@ class FieldList(Field):\n     @property\n     def data(self):\n         return [f.data for f in self.entries]\n+\n+\n+class SearchField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"search\">``.\n+    \"\"\"\n+\n+    widget = widgets.SearchInput()\n+\n+\n+class TelField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"tel\">``.\n+    \"\"\"\n+\n+    widget = widgets.TelInput()\n+\n+\n+class URLField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"url\">``.\n+    \"\"\"\n+\n+    widget = widgets.URLInput()\n+\n+\n+class EmailField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"email\">``.\n+    \"\"\"\n+\n+    widget = widgets.EmailInput()\n+\n+\n+class DateTimeLocalField(DateTimeField):\n+    \"\"\"\n+    Represents an ``<input type=\"datetime-local\">``.\n+    \"\"\"\n+\n+    widget = widgets.DateTimeLocalInput()\n+\n+\n+class IntegerRangeField(IntegerField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput()\n+\n+\n+class DecimalRangeField(DecimalField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/fields/html5.py b/src/wtforms/fields/html5.py\ndeleted file mode 100644\nindex 26d7efa..0000000\n--- a/src/wtforms/fields/html5.py\n+++ /dev/null\n@@ -1,121 +0,0 @@\n-from . import core\n-from ..widgets import html5 as widgets\n-\n-__all__ = (\n-    \"DateField\",\n-    \"DateTimeField\",\n-    \"DateTimeLocalField\",\n-    \"DecimalField\",\n-    \"DecimalRangeField\",\n-    \"EmailField\",\n-    \"IntegerField\",\n-    \"IntegerRangeField\",\n-    \"SearchField\",\n-    \"TelField\",\n-    \"TimeField\",\n-    \"URLField\",\n-)\n-\n-\n-class SearchField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"search\">``.\n-    \"\"\"\n-\n-    widget = widgets.SearchInput()\n-\n-\n-class TelField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"tel\">``.\n-    \"\"\"\n-\n-    widget = widgets.TelInput()\n-\n-\n-class URLField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"url\">``.\n-    \"\"\"\n-\n-    widget = widgets.URLInput()\n-\n-\n-class EmailField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"email\">``.\n-    \"\"\"\n-\n-    widget = widgets.EmailInput()\n-\n-\n-class DateTimeField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeInput()\n-\n-\n-class DateField(core.DateField):\n-    \"\"\"\n-    Represents an ``<input type=\"date\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateInput()\n-\n-\n-class TimeField(core.TimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"time\">``.\n-    \"\"\"\n-\n-    widget = widgets.TimeInput()\n-\n-\n-class MonthField(core.MonthField):\n-    \"\"\"\n-    Represents an ``<input type=\"month\">``.\n-    \"\"\"\n-\n-    widget = widgets.MonthInput()\n-\n-\n-class DateTimeLocalField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime-local\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeLocalInput()\n-\n-\n-class IntegerField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput()\n-\n-\n-class DecimalField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput(step=\"any\")\n-\n-\n-class IntegerRangeField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput()\n-\n-\n-class DecimalRangeField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/validators.py b/src/wtforms/validators.py\nindex 2d6bea2..059f32c 100644\n--- a/src/wtforms/validators.py\n+++ b/src/wtforms/validators.py\n@@ -114,6 +114,8 @@ class Length:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)d` and `%(max)d` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `minlength` and `maxlength` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=-1, max=-1, message=None):\n@@ -124,6 +126,11 @@ class Length:\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min != -1:\n+            self.field_flags[\"minlength\"] = self.min\n+        if self.max != -1:\n+            self.field_flags[\"maxlength\"] = self.max\n \n     def __call__(self, form, field):\n         length = field.data and len(field.data) or 0\n@@ -174,12 +181,19 @@ class NumberRange:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)s` and `%(max)s` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `min` and `max` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=None, max=None, message=None):\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min is not None:\n+            self.field_flags[\"min\"] = self.min\n+        if self.max is not None:\n+            self.field_flags[\"max\"] = self.max\n \n     def __call__(self, form, field):\n         data = field.data\n@@ -215,9 +229,9 @@ class Optional:\n     :param strip_whitespace:\n         If True (the default) also stop the validation chain on input which\n         consists of only whitespace.\n-    \"\"\"\n \n-    field_flags = (\"optional\",)\n+    Sets the `optional` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, strip_whitespace=True):\n         if strip_whitespace:\n@@ -225,6 +239,8 @@ class Optional:\n         else:\n             self.string_check = lambda s: s\n \n+        self.field_flags = {\"optional\": True}\n+\n     def __call__(self, form, field):\n         if (\n             not field.raw_data\n@@ -256,12 +272,13 @@ class DataRequired:\n \n     :param message:\n         Error message to raise in case of a validation error.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.data or isinstance(field.data, str) and not field.data.strip():\n@@ -281,12 +298,13 @@ class InputRequired:\n     Note there is a distinction between this and DataRequired in that\n     InputRequired looks that form-input data was provided, and DataRequired\n     looks at the post-coercion data.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.raw_data or not field.raw_data[0]:\ndiff --git a/src/wtforms/widgets/core.py b/src/wtforms/widgets/core.py\nindex 3e2a2eb..a0e0e61 100644\n--- a/src/wtforms/widgets/core.py\n+++ b/src/wtforms/widgets/core.py\n@@ -3,17 +3,30 @@ from markupsafe import Markup\n \n __all__ = (\n     \"CheckboxInput\",\n+    \"ColorInput\",\n+    \"DateInput\",\n+    \"DateTimeInput\",\n+    \"DateTimeLocalInput\",\n+    \"EmailInput\",\n     \"FileInput\",\n     \"HiddenInput\",\n     \"ListWidget\",\n+    \"MonthInput\",\n+    \"NumberInput\",\n+    \"Option\",\n     \"PasswordInput\",\n     \"RadioInput\",\n+    \"RangeInput\",\n+    \"SearchInput\",\n     \"Select\",\n     \"SubmitInput\",\n     \"TableWidget\",\n     \"TextArea\",\n     \"TextInput\",\n-    \"Option\",\n+    \"TelInput\",\n+    \"TimeInput\",\n+    \"URLInput\",\n+    \"WeekInput\",\n )\n \n \n@@ -148,6 +161,7 @@ class Input:\n     \"\"\"\n \n     html_params = staticmethod(html_params)\n+    validation_attrs = [\"required\"]\n \n     def __init__(self, input_type=None):\n         if input_type is not None:\n@@ -158,8 +172,10 @@ class Input:\n         kwargs.setdefault(\"type\", self.input_type)\n         if \"value\" not in kwargs:\n             kwargs[\"value\"] = field._value()\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\"<input %s>\" % self.html_params(name=field.name, **kwargs))\n \n \n@@ -169,6 +185,7 @@ class TextInput(Input):\n     \"\"\"\n \n     input_type = \"text\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n \n class PasswordInput(Input):\n@@ -181,6 +198,7 @@ class PasswordInput(Input):\n     \"\"\"\n \n     input_type = \"password\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n     def __init__(self, hide_value=True):\n         self.hide_value = hide_value\n@@ -196,9 +214,12 @@ class HiddenInput(Input):\n     Render a hidden input.\n     \"\"\"\n \n-    field_flags = (\"hidden\",)\n     input_type = \"hidden\"\n \n+    def __init__(self, *args, **kwargs):\n+        super().__init__(*args, **kwargs)\n+        self.field_flags = {\"hidden\": True}\n+\n \n class CheckboxInput(Input):\n     \"\"\"\n@@ -238,6 +259,7 @@ class FileInput(Input):\n     \"\"\"\n \n     input_type = \"file\"\n+    validation_attrs = [\"required\", \"accept\"]\n \n     def __init__(self, multiple=False):\n         super().__init__()\n@@ -275,10 +297,14 @@ class TextArea:\n     `rows` and `cols` ought to be passed as keyword args when rendering.\n     \"\"\"\n \n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\"]\n+\n     def __call__(self, field, **kwargs):\n         kwargs.setdefault(\"id\", field.id)\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\n             \"<textarea %s>\\r\\n%s</textarea>\"\n             % (html_params(name=field.name, **kwargs), escape(field._value()))\n@@ -297,6 +323,8 @@ class Select:\n     `(value, label, selected)`.\n     \"\"\"\n \n+    validation_attrs = [\"required\"]\n+\n     def __init__(self, multiple=False):\n         self.multiple = multiple\n \n@@ -304,8 +332,10 @@ class Select:\n         kwargs.setdefault(\"id\", field.id)\n         if self.multiple:\n             kwargs[\"multiple\"] = True\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         html = [\"<select %s>\" % html_params(name=field.name, **kwargs)]\n         for val, label, selected in field.iter_choices():\n             html.append(self.render_option(val, label, selected))\n@@ -338,3 +368,141 @@ class Option:\n         return Select.render_option(\n             field._value(), field.label.text, field.checked, **kwargs\n         )\n+\n+\n+class SearchInput(Input):\n+    \"\"\"\n+    Renders an input with type \"search\".\n+    \"\"\"\n+\n+    input_type = \"search\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class TelInput(Input):\n+    \"\"\"\n+    Renders an input with type \"tel\".\n+    \"\"\"\n+\n+    input_type = \"tel\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class URLInput(Input):\n+    \"\"\"\n+    Renders an input with type \"url\".\n+    \"\"\"\n+\n+    input_type = \"url\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class EmailInput(Input):\n+    \"\"\"\n+    Renders an input with type \"email\".\n+    \"\"\"\n+\n+    input_type = \"email\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class DateTimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime\".\n+    \"\"\"\n+\n+    input_type = \"datetime\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateInput(Input):\n+    \"\"\"\n+    Renders an input with type \"date\".\n+    \"\"\"\n+\n+    input_type = \"date\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class MonthInput(Input):\n+    \"\"\"\n+    Renders an input with type \"month\".\n+    \"\"\"\n+\n+    input_type = \"month\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class WeekInput(Input):\n+    \"\"\"\n+    Renders an input with type \"week\".\n+    \"\"\"\n+\n+    input_type = \"week\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class TimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"time\".\n+    \"\"\"\n+\n+    input_type = \"time\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateTimeLocalInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime-local\".\n+    \"\"\"\n+\n+    input_type = \"datetime-local\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class NumberInput(Input):\n+    \"\"\"\n+    Renders an input with type \"number\".\n+    \"\"\"\n+\n+    input_type = \"number\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None, min=None, max=None):\n+        self.step = step\n+        self.min = min\n+        self.max = max\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        if self.min is not None:\n+            kwargs.setdefault(\"min\", self.min)\n+        if self.max is not None:\n+            kwargs.setdefault(\"max\", self.max)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class RangeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"range\".\n+    \"\"\"\n+\n+    input_type = \"range\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None):\n+        self.step = step\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class ColorInput(Input):\n+    \"\"\"\n+    Renders an input with type \"color\".\n+    \"\"\"\n+\n+    input_type = \"color\"\ndiff --git a/src/wtforms/widgets/html5.py b/src/wtforms/widgets/html5.py\ndeleted file mode 100644\nindex 2a9ea76..0000000\n--- a/src/wtforms/widgets/html5.py\n+++ /dev/null\n@@ -1,143 +0,0 @@\n-from .core import Input\n-\n-__all__ = (\n-    \"ColorInput\",\n-    \"DateInput\",\n-    \"DateTimeInput\",\n-    \"DateTimeLocalInput\",\n-    \"EmailInput\",\n-    \"MonthInput\",\n-    \"NumberInput\",\n-    \"RangeInput\",\n-    \"SearchInput\",\n-    \"TelInput\",\n-    \"TimeInput\",\n-    \"URLInput\",\n-    \"WeekInput\",\n-)\n-\n-\n-class SearchInput(Input):\n-    \"\"\"\n-    Renders an input with type \"search\".\n-    \"\"\"\n-\n-    input_type = \"search\"\n-\n-\n-class TelInput(Input):\n-    \"\"\"\n-    Renders an input with type \"tel\".\n-    \"\"\"\n-\n-    input_type = \"tel\"\n-\n-\n-class URLInput(Input):\n-    \"\"\"\n-    Renders an input with type \"url\".\n-    \"\"\"\n-\n-    input_type = \"url\"\n-\n-\n-class EmailInput(Input):\n-    \"\"\"\n-    Renders an input with type \"email\".\n-    \"\"\"\n-\n-    input_type = \"email\"\n-\n-\n-class DateTimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime\".\n-    \"\"\"\n-\n-    input_type = \"datetime\"\n-\n-\n-class DateInput(Input):\n-    \"\"\"\n-    Renders an input with type \"date\".\n-    \"\"\"\n-\n-    input_type = \"date\"\n-\n-\n-class MonthInput(Input):\n-    \"\"\"\n-    Renders an input with type \"month\".\n-    \"\"\"\n-\n-    input_type = \"month\"\n-\n-\n-class WeekInput(Input):\n-    \"\"\"\n-    Renders an input with type \"week\".\n-    \"\"\"\n-\n-    input_type = \"week\"\n-\n-\n-class TimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"time\".\n-    \"\"\"\n-\n-    input_type = \"time\"\n-\n-\n-class DateTimeLocalInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime-local\".\n-    \"\"\"\n-\n-    input_type = \"datetime-local\"\n-\n-\n-class NumberInput(Input):\n-    \"\"\"\n-    Renders an input with type \"number\".\n-    \"\"\"\n-\n-    input_type = \"number\"\n-\n-    def __init__(self, step=None, min=None, max=None):\n-        self.step = step\n-        self.min = min\n-        self.max = max\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        if self.min is not None:\n-            kwargs.setdefault(\"min\", self.min)\n-        if self.max is not None:\n-            kwargs.setdefault(\"max\", self.max)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class RangeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"range\".\n-    \"\"\"\n-\n-    input_type = \"range\"\n-\n-    def __init__(self, step=None):\n-        self.step = step\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class ColorInput(Input):\n-    \"\"\"\n-    Renders an input with type \"color\".\n-    \"\"\"\n-\n-    input_type = \"color\"\n",
        "pr_description": "HTML5 widgets by default\nThis patch is based upon #467, the changes are in the last commit 44a1cec.\r\nIt fixes #594 \r\n",
        "problem_statement": "Use HTML 5 widgets\nThe HTML 5 widgets are kept in their own section and not used by default, but this distinction doesn't make sense today. \"HTML 5\" is just \"HTML\" now. No supported browser doesn't understand those input types, and ones that aren't supported fall back to text anyway.\r\n\r\nI already sort of started this by making the required validators add the `required` flag, and I'm taking it further with #406 for more complex flags. Fields should default to using the more specific widgets where possible.",
        "repo": "wtforms/wtforms",
        "test_patch": "diff --git a/tests/test_fields.py b/tests/test_fields.py\nindex f36ce71..0986534 100644\n--- a/tests/test_fields.py\n+++ b/tests/test_fields.py\n@@ -119,7 +119,7 @@ class TestFlags:\n     def test_existing_values(self, flags):\n         assert flags.required is True\n         assert \"required\" in flags\n-        assert flags.optional is False\n+        assert flags.optional is None\n         assert \"optional\" not in flags\n \n     def test_assignment(self, flags):\n@@ -1373,3 +1373,111 @@ class TestHTML5Fields:\n             if field.data != item[\"data\"]:\n                 tmpl = \"Field {key} data mismatch: {field.data!r} != {data!r}\"\n                 raise AssertionError(tmpl.format(field=field, **item))\n+\n+\n+class TestFieldValidatorsTest:\n+    class F(Form):\n+        v1 = [validators.Length(min=1)]\n+        v2 = [validators.Length(min=1, max=3)]\n+        string1 = StringField(validators=v1)\n+        string2 = StringField(validators=v2)\n+        password1 = PasswordField(validators=v1)\n+        password2 = PasswordField(validators=v2)\n+        textarea1 = TextAreaField(validators=v1)\n+        textarea2 = TextAreaField(validators=v2)\n+        search1 = SearchField(validators=v1)\n+        search2 = SearchField(validators=v2)\n+\n+        v3 = [validators.NumberRange(min=1)]\n+        v4 = [validators.NumberRange(min=1, max=3)]\n+        integer1 = IntegerField(validators=v3)\n+        integer2 = IntegerField(validators=v4)\n+        integerrange1 = IntegerRangeField(validators=v3)\n+        integerrange2 = IntegerRangeField(validators=v4)\n+        decimal1 = DecimalField(validators=v3)\n+        decimal2 = DecimalField(validators=v4)\n+        decimalrange1 = DecimalRangeField(validators=v3)\n+        decimalrange2 = DecimalRangeField(validators=v4)\n+\n+    def test_minlength_maxlength(self):\n+        form = self.F()\n+        assert (\n+            form.string1()\n+            == '<input id=\"string1\" minlength=\"1\" name=\"string1\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.string2() == '<input id=\"string2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"string2\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password1() == '<input id=\"password1\" minlength=\"1\"'\n+            ' name=\"password1\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password2() == '<input id=\"password2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"password2\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.textarea1() == '<textarea id=\"textarea1\" minlength=\"1\"'\n+            ' name=\"textarea1\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.textarea2() == '<textarea id=\"textarea2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"textarea2\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.search1() == '<input id=\"search1\" minlength=\"1\"'\n+            ' name=\"search1\" type=\"search\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.search2() == '<input id=\"search2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"search2\" type=\"search\" value=\"\">'\n+        )\n+\n+    def test_min_max(self):\n+        form = self.F()\n+        assert (\n+            form.integer1()\n+            == '<input id=\"integer1\" min=\"1\" name=\"integer1\" type=\"number\" value=\"\">'\n+        )\n+        assert (\n+            form.integer2() == '<input id=\"integer2\" max=\"3\" min=\"1\"'\n+            ' name=\"integer2\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange1() == '<input id=\"integerrange1\" min=\"1\"'\n+            ' name=\"integerrange1\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange2() == '<input id=\"integerrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"integerrange2\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal1() == '<input id=\"decimal1\" min=\"1\"'\n+            ' name=\"decimal1\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal2() == '<input id=\"decimal2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimal2\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange1() == '<input id=\"decimalrange1\" min=\"1\"'\n+            ' name=\"decimalrange1\" step=\"any\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange2() == '<input id=\"decimalrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimalrange2\" step=\"any\" type=\"range\" value=\"\">'\n+        )\ndiff --git a/tests/test_validators.py b/tests/test_validators.py\nindex dd4613c..4e373ff 100644\n--- a/tests/test_validators.py\n+++ b/tests/test_validators.py\n@@ -31,7 +31,7 @@ def test_data_required(dummy_form, dummy_field):\n     validator = data_required()\n     dummy_field.data = \"foobar\"\n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n @pytest.mark.parametrize(\"bad_val\", [None, \"\", \" \", \"\\t\\t\"])\n@@ -85,7 +85,7 @@ def test_input_required(dummy_form, dummy_field):\n     dummy_field.raw_data = [\"foobar\"]\n \n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n def test_input_required_raises(dummy_form, dummy_field):\n@@ -141,7 +141,7 @@ def test_input_optional_raises(data_v, raw_data_v, dummy_form, dummy_field):\n \n     with pytest.raises(StopValidation):\n         validator(dummy_form, dummy_field)\n-        assert validator.field_flags == (\"optional\",)\n+        assert validator.field_flags == {\"optional\": True}\n \n     dummy_field.errors = [\"Invalid Integer Value\"]\n     assert len(dummy_field.errors) == 1\ndiff --git a/tests/test_widgets.py b/tests/test_widgets.py\nindex 9bedec6..a9b7cd3 100644\n--- a/tests/test_widgets.py\n+++ b/tests/test_widgets.py\n@@ -7,14 +7,14 @@ from wtforms.widgets.core import HiddenInput\n from wtforms.widgets.core import html_params\n from wtforms.widgets.core import Input\n from wtforms.widgets.core import ListWidget\n+from wtforms.widgets.core import NumberInput\n from wtforms.widgets.core import PasswordInput\n from wtforms.widgets.core import RadioInput\n+from wtforms.widgets.core import RangeInput\n from wtforms.widgets.core import Select\n from wtforms.widgets.core import TableWidget\n from wtforms.widgets.core import TextArea\n from wtforms.widgets.core import TextInput\n-from wtforms.widgets.html5 import NumberInput\n-from wtforms.widgets.html5 import RangeInput\n \n \n class TestHTMLParams:\n",
        "FAIL_TO_PASS": [
          "tests/test_fields.py::TestDefaults::test",
          "tests/test_fields.py::TestLabel::test",
          "tests/test_fields.py::TestLabel::test_auto_label",
          "tests/test_fields.py::TestLabel::test_override_for",
          "tests/test_fields.py::TestLabel::test_escaped_label_text",
          "tests/test_fields.py::TestFlags::test_existing_values",
          "tests/test_fields.py::TestFlags::test_assignment",
          "tests/test_fields.py::TestFlags::test_unset",
          "tests/test_fields.py::TestFlags::test_repr",
          "tests/test_fields.py::TestFlags::test_underscore_property",
          "tests/test_fields.py::TestUnsetValue::test",
          "tests/test_fields.py::TestFilters::test_working",
          "tests/test_fields.py::TestFilters::test_failure",
          "tests/test_fields.py::TestFilters::test_extra",
          "tests/test_fields.py::TestFilters::test_inline",
          "tests/test_fields.py::TestField::test_unbound_field",
          "tests/test_fields.py::TestField::test_htmlstring",
          "tests/test_fields.py::TestField::test_str_coerce",
          "tests/test_fields.py::TestField::test_unicode_coerce",
          "tests/test_fields.py::TestField::test_process_formdata",
          "tests/test_fields.py::TestField::test_meta_attribute",
          "tests/test_fields.py::TestField::test_render_kw",
          "tests/test_fields.py::TestField::test_render_special",
          "tests/test_fields.py::TestField::test_select_field_copies_choices",
          "tests/test_fields.py::TestField::test_required_flag",
          "tests/test_fields.py::TestField::test_check_validators",
          "tests/test_fields.py::TestField::test_custom_name",
          "tests/test_fields.py::TestPrePostValidation::test_pre_stop",
          "tests/test_fields.py::TestPrePostValidation::test_post",
          "tests/test_fields.py::TestSelectField::test_defaults",
          "tests/test_fields.py::TestSelectField::test_with_data",
          "tests/test_fields.py::TestSelectField::test_value_coercion",
          "tests/test_fields.py::TestSelectField::test_iterable_options",
          "tests/test_fields.py::TestSelectField::test_default_coerce",
          "tests/test_fields.py::TestSelectField::test_validate_choices",
          "tests/test_fields.py::TestSelectField::test_validate_choices_when_empty",
          "tests/test_fields.py::TestSelectField::test_validate_choices_when_none",
          "tests/test_fields.py::TestSelectField::test_dont_validate_choices",
          "tests/test_fields.py::TestSelectField::test_choice_shortcut",
          "tests/test_fields.py::TestSelectField::test_empty_choice[choices0]",
          "tests/test_fields.py::TestSelectField::test_empty_choice[None]",
          "tests/test_fields.py::TestSelectField::test_callable_choices",
          "tests/test_fields.py::TestSelectMultipleField::test_defaults",
          "tests/test_fields.py::TestSelectMultipleField::test_with_data",
          "tests/test_fields.py::TestSelectMultipleField::test_coerce_fail",
          "tests/test_fields.py::TestSelectMultipleField::test_callable_choices",
          "tests/test_fields.py::TestSelectMultipleField::test_choice_shortcut",
          "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[choices0]",
          "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[None]",
          "tests/test_fields.py::TestRadioField::test",
          "tests/test_fields.py::TestRadioField::test_text_coercion",
          "tests/test_fields.py::TestRadioField::test_callable_choices",
          "tests/test_fields.py::TestStringField::test",
          "tests/test_fields.py::TestHiddenField::test",
          "tests/test_fields.py::TestTextAreaField::test",
          "tests/test_fields.py::TestPasswordField::test",
          "tests/test_fields.py::TestFileField::test_file_field",
          "tests/test_fields.py::TestFileField::test_multiple_file_field",
          "tests/test_fields.py::TestFileField::test_file_field_without_file_input",
          "tests/test_fields.py::TestIntegerField::test",
          "tests/test_fields.py::TestDecimalField::test",
          "tests/test_fields.py::TestDecimalField::test_quantize",
          "tests/test_fields.py::TestFloatField::test",
          "tests/test_fields.py::TestBooleanField::test_defaults",
          "tests/test_fields.py::TestBooleanField::test_rendering",
          "tests/test_fields.py::TestBooleanField::test_with_postdata",
          "tests/test_fields.py::TestBooleanField::test_with_model_data",
          "tests/test_fields.py::TestBooleanField::test_with_postdata_and_model",
          "tests/test_fields.py::TestDateField::test_basic",
          "tests/test_fields.py::TestDateField::test_failure",
          "tests/test_fields.py::TestMonthField::test_basic",
          "tests/test_fields.py::TestMonthField::test_failure",
          "tests/test_fields.py::TestTimeField::test_basic",
          "tests/test_fields.py::TestDateTimeField::test_basic",
          "tests/test_fields.py::TestDateTimeField::test_microseconds",
          "tests/test_fields.py::TestSubmitField::test",
          "tests/test_fields.py::TestFormField::test_formdata",
          "tests/test_fields.py::TestFormField::test_iteration",
          "tests/test_fields.py::TestFormField::test_with_obj",
          "tests/test_fields.py::TestFormField::test_widget",
          "tests/test_fields.py::TestFormField::test_separator",
          "tests/test_fields.py::TestFormField::test_no_validators_or_filters",
          "tests/test_fields.py::TestFormField::test_populate_missing_obj",
          "tests/test_fields.py::TestFieldList::test_form",
          "tests/test_fields.py::TestFieldList::test_enclosed_subform",
          "tests/test_fields.py::TestFieldList::test_enclosed_subform_custom_name",
          "tests/test_fields.py::TestFieldList::test_entry_management",
          "tests/test_fields.py::TestFieldList::test_min_max_entries",
          "tests/test_fields.py::TestFieldList::test_validators",
          "tests/test_fields.py::TestFieldList::test_no_filters",
          "tests/test_fields.py::TestFieldList::test_process_prefilled",
          "tests/test_fields.py::TestFieldList::test_errors",
          "tests/test_fields.py::TestCustomFieldQuirks::test_processing_failure",
          "tests/test_fields.py::TestCustomFieldQuirks::test_default_impls",
          "tests/test_fields.py::TestHTML5Fields::test_simple",
          "tests/test_fields.py::TestFieldValidatorsTest::test_minlength_maxlength",
          "tests/test_fields.py::TestFieldValidatorsTest::test_min_max",
          "tests/test_validators.py::test_data_required",
          "tests/test_validators.py::test_data_required_raises[None]",
          "tests/test_validators.py::test_data_required_raises[]",
          "tests/test_validators.py::test_data_required_raises[",
          "tests/test_validators.py::test_data_required_raises[\\t\\t]",
          "tests/test_validators.py::test_data_required_clobber",
          "tests/test_validators.py::test_data_required_messages[validator0-This",
          "tests/test_validators.py::test_data_required_messages[validator1-foo]",
          "tests/test_validators.py::test_input_required",
          "tests/test_validators.py::test_input_required_raises",
          "tests/test_validators.py::test_input_required_error_message[validator0-This",
          "tests/test_validators.py::test_input_required_error_message[validator1-foo]",
          "tests/test_validators.py::test_input_optional_passes",
          "tests/test_validators.py::test_input_optional_raises[-]",
          "tests/test_validators.py::test_input_optional_raises[",
          "tests/test_validators.py::test_input_optional_raises[\\t-\\t]",
          "tests/test_validators.py::test_ip4address_passes[147.230.23.25]",
          "tests/test_validators.py::test_ip4address_passes[147.230.23.0]",
          "tests/test_validators.py::test_ip4address_passes[127.0.0.1]",
          "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::1111]",
          "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::]",
          "tests/test_validators.py::test_good_ip6address_passes[::1]",
          "tests/test_validators.py::test_good_ip6address_passes[dead:beef:0:0:0:0:42:1]",
          "tests/test_validators.py::test_good_ip6address_passes[abcd:ef::42:1]",
          "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::1111]",
          "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::]",
          "tests/test_validators.py::test_bad_ip6address_raises[abc.0.0.1]",
          "tests/test_validators.py::test_bad_ip6address_raises[abcd:1234::123::1]",
          "tests/test_validators.py::test_bad_ip6address_raises[1:2:3:4:5:6:7:8:9]",
          "tests/test_validators.py::test_bad_ip6address_raises[abcd::1ffff]",
          "tests/test_validators.py::test_bad_ip4address_raises[147.230.1000.25]",
          "tests/test_validators.py::test_bad_ip4address_raises[2001:718::::]",
          "tests/test_validators.py::test_bad_ip4address_raises[abc.0.0.1]",
          "tests/test_validators.py::test_bad_ip4address_raises[1278.0.0.1]",
          "tests/test_validators.py::test_bad_ip4address_raises[127.0.0.abc]",
          "tests/test_validators.py::test_bad_ip4address_raises[900.200.100.75]",
          "tests/test_validators.py::test_valid_email_passes[foo@bar.dk]",
          "tests/test_validators.py::test_valid_email_passes[123@bar.dk]",
          "tests/test_validators.py::test_valid_email_passes[foo@456.dk]",
          "tests/test_validators.py::test_valid_email_passes[foo@bar456.info]",
          "tests/test_validators.py::test_valid_email_passes[foo@b\\xfccher.\\u4e2d\\u56fd]",
          "tests/test_validators.py::test_invalid_email_raises[None]",
          "tests/test_validators.py::test_invalid_email_raises[]",
          "tests/test_validators.py::test_invalid_email_raises[",
          "tests/test_validators.py::test_invalid_email_raises[foo]",
          "tests/test_validators.py::test_invalid_email_raises[bar.dk]",
          "tests/test_validators.py::test_invalid_email_raises[foo@]",
          "tests/test_validators.py::test_invalid_email_raises[@bar.dk]",
          "tests/test_validators.py::test_invalid_email_raises[foo@bar]",
          "tests/test_validators.py::test_invalid_email_raises[foo@bar.ab12]",
          "tests/test_validators.py::test_invalid_email_raises[foo@.bar.ab]",
          "tests/test_validators.py::test_invalid_email_raises[foo.@bar.co]",
          "tests/test_validators.py::test_invalid_email_raises[foo@foo@bar.co]",
          "tests/test_validators.py::test_invalid_email_raises[fo",
          "tests/test_validators.py::test_invalid_email_raises_granular_message[foo@]",
          "tests/test_validators.py::test_ip_address_raises_on_bad_init",
          "tests/test_validators.py::test_valid_mac_passes[01:23:45:67:ab:CD]",
          "tests/test_validators.py::test_bad_mac_raises[00:00:00:00:00]",
          "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:]",
          "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:gh]",
          "tests/test_validators.py::test_bad_mac_raises[123:23:45:67:89:00]",
          "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f-0deb-43e9-92a1-4775189ec9f8]",
          "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f0deb43e992a14775189ec9f8]",
          "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-deb-43e9-92a1-4775189ec9f8]",
          "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-0deb-43e9-92a1-4775189ec9f]",
          "tests/test_validators.py::test_bad_uuid_raises[gbc1c94f-0deb-43e9-92a1-4775189ec9f8]",
          "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f",
          "tests/test_validators.py::test_required_passes",
          "tests/test_validators.py::test_required_stops_validation",
          "tests/test_validators.py::test_equal_to_passes",
          "tests/test_validators.py::test_equal_to_raises[test-invalid_field_name]",
          "tests/test_validators.py::test_equal_to_raises[bad_value-foo]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.dk]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/]",
          "tests/test_validators.py::test_valid_url_passes[http://foo-bar.dk/]",
          "tests/test_validators.py::test_valid_url_passes[http://foo--bar.dk/]",
          "tests/test_validators.py::test_valid_url_passes[http://foo_bar.dk/]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.dk?query=param]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param&foo=faa]",
          "tests/test_validators.py::test_valid_url_passes[http://foobar.museum/foobar]",
          "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1/foobar]",
          "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1:9000/fake]",
          "tests/test_validators.py::test_valid_url_passes[http://\\u0645\\u062b\\u0627\\u0644.\\u0625\\u062e\\u062a\\u0628\\u0627\\u0631/foo.com]",
          "tests/test_validators.py::test_valid_url_passes[http://\\u0909\\u0926\\u093e\\u0939\\u0930\\u0923.\\u092a\\u0930\\u0940\\u0915\\u094d\\u0937\\u093e/]",
          "tests/test_validators.py::test_valid_url_passes[http://\\uc2e4\\ub840.\\ud14c\\uc2a4\\ud2b8]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://localhost/foobar]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://foobar]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://foobar?query=param&foo=faa]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:5000?query=param&foo=faa]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://foobar/path?query=param&foo=faa]",
          "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:1234/path?query=param&foo=faa]",
          "tests/test_validators.py::test_bad_url_raises[http://foobar]",
          "tests/test_validators.py::test_bad_url_raises[http://-foobar.dk/]",
          "tests/test_validators.py::test_bad_url_raises[http://foobar-.dk/]",
          "tests/test_validators.py::test_bad_url_raises[foobar.dk]",
          "tests/test_validators.py::test_bad_url_raises[http://127.0.0/asdf]",
          "tests/test_validators.py::test_bad_url_raises[http://foobar.d]",
          "tests/test_validators.py::test_bad_url_raises[http://foobar.12]",
          "tests/test_validators.py::test_bad_url_raises[http://localhost:abc/a]",
          "tests/test_validators.py::test_anyof_passes[b-test_list0]",
          "tests/test_validators.py::test_anyof_passes[2-test_list1]",
          "tests/test_validators.py::test_anyof_raisses[d-test_list0]",
          "tests/test_validators.py::test_anyof_raisses[6-test_list1]",
          "tests/test_validators.py::test_any_of_values_formatter",
          "tests/test_validators.py::test_none_of_passes",
          "tests/test_validators.py::test_none_of_raises",
          "tests/test_validators.py::test_correct_length_passes[2-6]",
          "tests/test_validators.py::test_correct_length_passes[6--1]",
          "tests/test_validators.py::test_correct_length_passes[-1-6]",
          "tests/test_validators.py::test_correct_length_passes[6-6]",
          "tests/test_validators.py::test_bad_length_raises[7--1]",
          "tests/test_validators.py::test_bad_length_raises[-1-5]",
          "tests/test_validators.py::test_bad_length_init_raises[-1--1]",
          "tests/test_validators.py::test_bad_length_init_raises[5-2]",
          "tests/test_validators.py::test_length_messages[validator0-2",
          "tests/test_validators.py::test_length_messages[validator1-at",
          "tests/test_validators.py::test_length_messages[validator2-longer",
          "tests/test_validators.py::test_length_messages[validator3-between",
          "tests/test_validators.py::test_length_messages[validator4-exactly",
          "tests/test_validators.py::test_number_range_passes[5-10-7]",
          "tests/test_validators.py::test_number_range_passes[5-None-7]",
          "tests/test_validators.py::test_number_range_passes[None-100-70]",
          "tests/test_validators.py::test_number_range_raises[5-10-None]",
          "tests/test_validators.py::test_number_range_raises[5-10-0]",
          "tests/test_validators.py::test_number_range_raises[5-10-12]",
          "tests/test_validators.py::test_number_range_raises[5-10--5]",
          "tests/test_validators.py::test_number_range_raises[5-None-4]",
          "tests/test_validators.py::test_number_range_raises[None-100-500]",
          "tests/test_validators.py::test_number_range_nan[nan]",
          "tests/test_validators.py::test_number_range_nan[nan1]",
          "tests/test_validators.py::test_lazy_proxy_raises",
          "tests/test_validators.py::test_lazy_proxy_fixture",
          "tests/test_validators.py::test_regex_passes[^a-None-abcd-a0]",
          "tests/test_validators.py::test_regex_passes[^a-re.IGNORECASE-ABcd-A]",
          "tests/test_validators.py::test_regex_passes[^a-None-abcd-a1]",
          "tests/test_validators.py::test_regex_passes[^a-None-ABcd-A]",
          "tests/test_validators.py::test_regex_raises[^a-None-ABC]",
          "tests/test_validators.py::test_regex_raises[^a-re.IGNORECASE-foo]",
          "tests/test_validators.py::test_regex_raises[^a-None-None0]",
          "tests/test_validators.py::test_regex_raises[^a-None-foo]",
          "tests/test_validators.py::test_regex_raises[^a-None-None1]",
          "tests/test_validators.py::test_regexp_message",
          "tests/test_validators.py::test_raise_exceptions[IndexError]",
          "tests/test_validators.py::test_raise_exceptions[ZeroDivisionError]",
          "tests/test_validators.py::test_raise_exceptions[ValueError]",
          "tests/test_widgets.py::TestHTMLParams::test_basic",
          "tests/test_widgets.py::TestHTMLParams::test_data_prefix",
          "tests/test_widgets.py::TestHTMLParams::test_aria_prefix",
          "tests/test_widgets.py::TestHTMLParams::test_quoting",
          "tests/test_widgets.py::TestListWidget::test_listwidget",
          "tests/test_widgets.py::TestTableWidget::test_tablewidget",
          "tests/test_widgets.py::TestBasicWidgets::test_input_type",
          "tests/test_widgets.py::TestBasicWidgets::test_html_marking",
          "tests/test_widgets.py::TestBasicWidgets::test_text_input",
          "tests/test_widgets.py::TestBasicWidgets::test_password_input",
          "tests/test_widgets.py::TestBasicWidgets::test_hidden_input",
          "tests/test_widgets.py::TestBasicWidgets::test_checkbox_input",
          "tests/test_widgets.py::TestBasicWidgets::test_radio_input",
          "tests/test_widgets.py::TestBasicWidgets::test_textarea",
          "tests/test_widgets.py::TestBasicWidgets::test_file",
          "tests/test_widgets.py::TestSelect::test_select",
          "tests/test_widgets.py::TestSelect::test_render_option",
          "tests/test_widgets.py::TestHTML5::test_number",
          "tests/test_widgets.py::TestHTML5::test_range"
        ],
        "PASS_TO_PASS": [],
        "install_config": {
          "base_image_name": "python_base_310",
          "docker_specs": {
            "_variant": null,
            "bazel_version": null,
            "bun_version": null,
            "cargo_version": null,
            "deno_version": null,
            "docker_version": null,
            "erlang_version": null,
            "gcc_version": null,
            "go_version": null,
            "helm_version": null,
            "java_version": null,
            "jdk_version": null,
            "llvm_version": null,
            "lua_version": null,
            "luajit_version": null,
            "neovim_version": null,
            "node_version": null,
            "npm_version": null,
            "nvim_version": null,
            "pnpm_version": null,
            "python_image": "python_base_310",
            "python_version": null,
            "redis_version": null,
            "ruby_version": null,
            "rust_version": null,
            "rustc_version": null,
            "solana_version": null,
            "sqlite_version": null
          },
          "install": [
            "pip install -q -e .[email] pytest babel"
          ],
          "log_parser": "parse_log_pytest",
          "test_cmd": "pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning tests/test_fields.py tests/test_validators.py tests/test_widgets.py"
        },
        "meta": {
          "llm_metadata": {
            "code": "A",
            "confidence": 0.96,
            "detected_issues": {
              "B1": false,
              "B2": false,
              "B3": false,
              "B4": false,
              "B5": false,
              "B6": false
            },
            "difficulty": "medium",
            "external_urls": [],
            "intent_completeness": "complete",
            "pr_categories": [
              "core_feat"
            ],
            "reasoning": "The issue requests default use of HTML5 widgets and flag handling changes, which the test patch verifies by updating expectations for flags, widget imports, and field defaults. The tests now match these new requirements, showing alignment without contradictory or missing assertions. No signals of B-category problems (no unrelated test coupling, naming mismatches, external dependencies, ambiguous specs, extra patch artifacts, or hidden domain knowledge) are present. Therefore the task is well‑specified and solvable.",
            "test_alignment_issues": []
          },
          "num_modified_files": 8,
          "num_modified_lines": 340,
          "pr_author": "azmeuk",
          "pr_labels": [],
          "pr_url": null
        }
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "reference": {
        "answer": "Source gold repair patch (not executed in this gallery).",
        "solution": "diff --git a/CHANGES.rst b/CHANGES.rst\nindex 50f43a2..a199f46 100644\n--- a/CHANGES.rst\n+++ b/CHANGES.rst\n@@ -34,7 +34,7 @@ Unreleased\n -   Render attribute names like ``for_`` and ``class_`` are normalized\n     consistently so later values override those specified earlier.\n     :issue:`449`, :pr:`596`\n-\n+-   Flags can take non-boolean values. :issue:`406` :pr:`467`\n \n Version 2.3.1\n -------------\ndiff --git a/docs/fields.rst b/docs/fields.rst\nindex ed047ce..0bd3e32 100644\n--- a/docs/fields.rst\n+++ b/docs/fields.rst\n@@ -182,10 +182,10 @@ The Field base class\n \n     .. attribute:: flags\n \n-        An object containing boolean flags set either by the field itself, or\n+        An object containing flags set either by the field itself, or\n         by validators on the field. For example, the built-in\n         :class:`~wtforms.validators.InputRequired` validator sets the `required` flag.\n-        An unset flag will result in :const:`False`.\n+        An unset flag will result in :const:`None`.\n \n         .. code-block:: html+django\n \n@@ -220,10 +220,14 @@ refer to a single input from the form.\n \n .. autoclass:: DateTimeField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n-    For better date/time fields, see the :mod:`dateutil extension <wtforms.ext.dateutil.fields>`\n+.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n \n .. autoclass:: DecimalField(default field arguments, places=2, rounding=None, use_locale=False, number_format=None)\n \n+.. autoclass:: DecimalRangeField(default field arguments)\n+\n+.. autoclass:: EmailField(default field arguments)\n+\n .. autoclass:: FileField(default field arguments)\n \n     Example usage::\n@@ -252,6 +256,8 @@ refer to a single input from the form.\n \n .. autoclass:: IntegerField(default field arguments)\n \n+.. autoclass:: IntegerRangeField(default field arguments)\n+\n .. autoclass:: RadioField(default field arguments, choices=[], coerce=str)\n \n     .. code-block:: jinja\n@@ -323,6 +329,8 @@ refer to a single input from the form.\n     a list of fields each representing an option. The rendering of this can be\n     further controlled by specifying `option_widget=`.\n \n+.. autoclass:: SearchField(default field arguments)\n+\n .. autoclass:: SelectMultipleField(default field arguments, choices=[], coerce=str, option_widget=None)\n \n    The data on the SelectMultipleField is stored as a list of objects, each of\n@@ -338,6 +346,13 @@ refer to a single input from the form.\n \n         {{ form.username(size=30, maxlength=50) }}\n \n+.. autoclass:: TelField(default field arguments)\n+\n+.. autoclass:: TimeField(default field arguments, format='%H:%M')\n+\n+.. autoclass:: URLField(default field arguments)\n+\n+\n Convenience Fields\n ------------------\n \n@@ -559,40 +574,3 @@ Additional Helper Classes\n     .. attribute:: text\n \n         The original label text passed to the field's constructor.\n-\n-HTML5 Fields\n-------------\n-\n-In addition to basic HTML fields, WTForms also supplies fields for the HTML5\n-standard. These fields can be accessed under the :mod:`wtforms.fields.html5` namespace.\n-In reality, these fields are just convenience fields that extend basic fields\n-and implement HTML5 specific widgets. These widgets are located in the :mod:`wtforms.widgets.html5`\n-namespace and can be overridden or modified just like any other widget.\n-\n-.. module:: wtforms.fields.html5\n-\n-.. autoclass:: SearchField(default field arguments)\n-\n-.. autoclass:: TelField(default field arguments)\n-\n-.. autoclass:: URLField(default field arguments)\n-\n-.. autoclass:: EmailField(default field arguments)\n-\n-.. autoclass:: DateTimeField(default field arguments,  format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: DateField(default field arguments, format='%Y-%m-%d')\n-\n-.. autoclass:: MonthField(default field arguments, format='%Y-%m')\n-\n-.. autoclass:: TimeField(default field arguments, format='%H:%M')\n-\n-.. autoclass:: DateTimeLocalField(default field arguments, format='%Y-%m-%d %H:%M:%S')\n-\n-.. autoclass:: IntegerField(default field arguments)\n-\n-.. autoclass:: DecimalField(default field arguments)\n-\n-.. autoclass:: IntegerRangeField(default field arguments)\n-\n-.. autoclass:: DecimalRangeField(default field arguments)\ndiff --git a/docs/validators.rst b/docs/validators.rst\nindex cdb9627..818c018 100644\n--- a/docs/validators.rst\n+++ b/docs/validators.rst\n@@ -177,22 +177,21 @@ which will then be available on the :attr:`field's flags object\n \n To specify flags on your validator, set the ``field_flags`` attribute on your\n validator. When the Field is constructed, the flags with the same name will be\n-set to True on your field. For example, let's imagine a validator that\n+set on your field. For example, let's imagine a validator that\n validates that input is valid BBCode. We can set a flag on the field then to\n signify that the field accepts BBCode::\n \n     # class implementation\n     class ValidBBCode(object):\n-        field_flags = ('accepts_bbcode', )\n-\n-        pass # validator implementation here\n+        def __init__(self):\n+            self.field_flags = {'accepts_bbcode': True}\n \n     # factory implementation\n     def valid_bbcode():\n         def _valid_bbcode(form, field):\n             pass # validator implementation here\n \n-        _valid_bbcode.field_flags = ('accepts_bbcode', )\n+        _valid_bbcode.field_flags = {'accepts_bbcode': True}\n         return _valid_bbcode\n \n Then we can check it in our template, so we can then place a note to the user:\n@@ -206,7 +205,10 @@ Then we can check it in our template, so we can then place a note to the user:\n \n Some considerations on using flags:\n \n-* Flags can only set boolean values, and another validator cannot unset them.\n-* If multiple fields set the same flag, its value is still True.\n+* Boolean flags will set HTML valueless attributes (e.g. `{required: True}`\n+  will give `<input type=\"text\" required>`). Other flag types will set regular\n+  HTML attributes (e.g. `{maxlength: 8}` will give `<input type=\"text\" maxlength=\"8\">`).\n+* If multiple validators set the same flag, the flag will have the value set\n+  by the first validator.\n * Flags are set from validators only in :meth:`Field.__init__`, so inline\n   validators and extra passed-in validators cannot set them.\ndiff --git a/docs/widgets.rst b/docs/widgets.rst\nindex 81b96c0..5c4f009 100644\n--- a/docs/widgets.rst\n+++ b/docs/widgets.rst\n@@ -14,36 +14,30 @@ recognize as not needing to be auto-escaped.\n Built-in widgets\n ----------------\n \n-.. autoclass:: wtforms.widgets.ListWidget\n-.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.ColorInput\n+.. autoclass:: wtforms.widgets.CheckboxInput\n+.. autoclass:: wtforms.widgets.DateTimeInput\n+.. autoclass:: wtforms.widgets.DateTimeLocalInput\n+.. autoclass:: wtforms.widgets.DateInput\n+.. autoclass:: wtforms.widgets.EmailInput\n+.. autoclass:: wtforms.widgets.FileInput\n+.. autoclass:: wtforms.widgets.HiddenInput\n .. autoclass:: wtforms.widgets.Input\n-.. autoclass:: wtforms.widgets.TextInput()\n+.. autoclass:: wtforms.widgets.ListWidget\n+.. autoclass:: wtforms.widgets.MonthInput\n+.. autoclass:: wtforms.widgets.NumberInput\n .. autoclass:: wtforms.widgets.PasswordInput\n-.. autoclass:: wtforms.widgets.HiddenInput()\n-.. autoclass:: wtforms.widgets.CheckboxInput()\n-.. autoclass:: wtforms.widgets.FileInput()\n-.. autoclass:: wtforms.widgets.SubmitInput()\n-.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.RangeInput\n+.. autoclass:: wtforms.widgets.SubmitInput\n+.. autoclass:: wtforms.widgets.SearchInput\n .. autoclass:: wtforms.widgets.Select\n-\n-HTML5 widgets\n--------------\n-\n-.. module:: wtforms.widgets.html5\n-\n-.. autoclass:: wtforms.widgets.html5.ColorInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeInput\n-.. autoclass:: wtforms.widgets.html5.DateTimeLocalInput\n-.. autoclass:: wtforms.widgets.html5.DateInput\n-.. autoclass:: wtforms.widgets.html5.EmailInput\n-.. autoclass:: wtforms.widgets.html5.MonthInput\n-.. autoclass:: wtforms.widgets.html5.NumberInput\n-.. autoclass:: wtforms.widgets.html5.RangeInput\n-.. autoclass:: wtforms.widgets.html5.SearchInput\n-.. autoclass:: wtforms.widgets.html5.TelInput\n-.. autoclass:: wtforms.widgets.html5.TimeInput\n-.. autoclass:: wtforms.widgets.html5.URLInput\n-.. autoclass:: wtforms.widgets.html5.WeekInput\n+.. autoclass:: wtforms.widgets.TableWidget\n+.. autoclass:: wtforms.widgets.TelInput\n+.. autoclass:: wtforms.widgets.TextArea\n+.. autoclass:: wtforms.widgets.TextInput\n+.. autoclass:: wtforms.widgets.TimeInput\n+.. autoclass:: wtforms.widgets.URLInput\n+.. autoclass:: wtforms.widgets.WeekInput\n \n Widget-Building Utilities\n -------------------------\ndiff --git a/src/wtforms/fields/__init__.py b/src/wtforms/fields/__init__.py\nindex 1f770d3..e429631 100644\n--- a/src/wtforms/fields/__init__.py\n+++ b/src/wtforms/fields/__init__.py\n@@ -3,6 +3,5 @@ from wtforms.fields.core import Field\n from wtforms.fields.core import Flags\n from wtforms.fields.core import Label\n from wtforms.fields.core import SelectFieldBase\n-from wtforms.fields.html5 import *\n from wtforms.fields.simple import *\n from wtforms.utils import unset_value as _unset_value\ndiff --git a/src/wtforms/fields/core.py b/src/wtforms/fields/core.py\nindex d835092..8724356 100644\n--- a/src/wtforms/fields/core.py\n+++ b/src/wtforms/fields/core.py\n@@ -2,6 +2,7 @@ import datetime\n import decimal\n import inspect\n import itertools\n+import warnings\n \n from markupsafe import escape\n from markupsafe import Markup\n@@ -15,18 +16,25 @@ from wtforms.validators import ValidationError\n __all__ = (\n     \"BooleanField\",\n     \"DecimalField\",\n+    \"DecimalRangeField\",\n     \"DateField\",\n     \"DateTimeField\",\n+    \"DateTimeLocalField\",\n+    \"EmailField\",\n     \"FieldList\",\n     \"FloatField\",\n     \"FormField\",\n     \"IntegerField\",\n+    \"IntegerRangeField\",\n     \"RadioField\",\n+    \"SearchField\",\n     \"SelectField\",\n     \"SelectMultipleField\",\n     \"StringField\",\n+    \"TelField\",\n     \"TimeField\",\n     \"MonthField\",\n+    \"URLField\",\n )\n \n \n@@ -143,9 +151,21 @@ class Field:\n             self.widget = widget\n \n         for v in itertools.chain(self.validators, [self.widget]):\n-            flags = getattr(v, \"field_flags\", ())\n-            for f in flags:\n-                setattr(self.flags, f, True)\n+            flags = getattr(v, \"field_flags\", {})\n+\n+            # check for legacy format, remove eventually\n+            if isinstance(flags, tuple):\n+                warnings.warn(\n+                    \"Flags should be stored in dicts and not in tuples. \"\n+                    \"The next version of WTForms will abandon support \"\n+                    \"for flags in tuples.\",\n+                    DeprecationWarning,\n+                    stacklevel=2,\n+                )\n+                flags = {flag_name: True for flag_name in flags}\n+\n+            for k, v in flags.items():\n+                setattr(self.flags, k, v)\n \n     def __str__(self):\n         \"\"\"\n@@ -408,15 +428,15 @@ class UnboundField:\n \n class Flags:\n     \"\"\"\n-    Holds a set of boolean flags as attributes.\n+    Holds a set of flags as attributes.\n \n-    Accessing a non-existing attribute returns False for its value.\n+    Accessing a non-existing attribute returns None for its value.\n     \"\"\"\n \n     def __getattr__(self, name):\n         if name.startswith(\"_\"):\n             return super().__getattr__(name)\n-        return False\n+        return None\n \n     def __contains__(self, name):\n         return getattr(self, name)\n@@ -670,7 +690,7 @@ class IntegerField(Field):\n     is ignored and will not be accepted as a value.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput()\n \n     def __init__(self, label=None, validators=None, **kwargs):\n         super().__init__(label, validators, **kwargs)\n@@ -721,7 +741,7 @@ class DecimalField(LocaleAwareNumberField):\n         format for the locale.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.NumberInput(step=\"any\")\n \n     def __init__(\n         self, label=None, validators=None, places=unset_value, rounding=None, **kwargs\n@@ -842,7 +862,7 @@ class DateTimeField(Field):\n     A text field which stores a `datetime.datetime` matching a format.\n     \"\"\"\n \n-    widget = widgets.TextInput()\n+    widget = widgets.DateTimeInput()\n \n     def __init__(\n         self, label=None, validators=None, format=\"%Y-%m-%d %H:%M:%S\", **kwargs\n@@ -871,6 +891,8 @@ class DateField(DateTimeField):\n     Same as DateTimeField, except stores a `datetime.date`.\n     \"\"\"\n \n+    widget = widgets.DateInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m-%d\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -889,6 +911,8 @@ class TimeField(DateTimeField):\n     Same as DateTimeField, except stores a `time`.\n     \"\"\"\n \n+    widget = widgets.TimeInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%H:%M\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -908,6 +932,8 @@ class MonthField(DateField):\n     with `day = 1`.\n     \"\"\"\n \n+    widget = widgets.MonthInput()\n+\n     def __init__(self, label=None, validators=None, format=\"%Y-%m\", **kwargs):\n         super().__init__(label, validators, format, **kwargs)\n \n@@ -1189,3 +1215,59 @@ class FieldList(Field):\n     @property\n     def data(self):\n         return [f.data for f in self.entries]\n+\n+\n+class SearchField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"search\">``.\n+    \"\"\"\n+\n+    widget = widgets.SearchInput()\n+\n+\n+class TelField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"tel\">``.\n+    \"\"\"\n+\n+    widget = widgets.TelInput()\n+\n+\n+class URLField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"url\">``.\n+    \"\"\"\n+\n+    widget = widgets.URLInput()\n+\n+\n+class EmailField(StringField):\n+    \"\"\"\n+    Represents an ``<input type=\"email\">``.\n+    \"\"\"\n+\n+    widget = widgets.EmailInput()\n+\n+\n+class DateTimeLocalField(DateTimeField):\n+    \"\"\"\n+    Represents an ``<input type=\"datetime-local\">``.\n+    \"\"\"\n+\n+    widget = widgets.DateTimeLocalInput()\n+\n+\n+class IntegerRangeField(IntegerField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput()\n+\n+\n+class DecimalRangeField(DecimalField):\n+    \"\"\"\n+    Represents an ``<input type=\"range\">``.\n+    \"\"\"\n+\n+    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/fields/html5.py b/src/wtforms/fields/html5.py\ndeleted file mode 100644\nindex 26d7efa..0000000\n--- a/src/wtforms/fields/html5.py\n+++ /dev/null\n@@ -1,121 +0,0 @@\n-from . import core\n-from ..widgets import html5 as widgets\n-\n-__all__ = (\n-    \"DateField\",\n-    \"DateTimeField\",\n-    \"DateTimeLocalField\",\n-    \"DecimalField\",\n-    \"DecimalRangeField\",\n-    \"EmailField\",\n-    \"IntegerField\",\n-    \"IntegerRangeField\",\n-    \"SearchField\",\n-    \"TelField\",\n-    \"TimeField\",\n-    \"URLField\",\n-)\n-\n-\n-class SearchField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"search\">``.\n-    \"\"\"\n-\n-    widget = widgets.SearchInput()\n-\n-\n-class TelField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"tel\">``.\n-    \"\"\"\n-\n-    widget = widgets.TelInput()\n-\n-\n-class URLField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"url\">``.\n-    \"\"\"\n-\n-    widget = widgets.URLInput()\n-\n-\n-class EmailField(core.StringField):\n-    \"\"\"\n-    Represents an ``<input type=\"email\">``.\n-    \"\"\"\n-\n-    widget = widgets.EmailInput()\n-\n-\n-class DateTimeField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeInput()\n-\n-\n-class DateField(core.DateField):\n-    \"\"\"\n-    Represents an ``<input type=\"date\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateInput()\n-\n-\n-class TimeField(core.TimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"time\">``.\n-    \"\"\"\n-\n-    widget = widgets.TimeInput()\n-\n-\n-class MonthField(core.MonthField):\n-    \"\"\"\n-    Represents an ``<input type=\"month\">``.\n-    \"\"\"\n-\n-    widget = widgets.MonthInput()\n-\n-\n-class DateTimeLocalField(core.DateTimeField):\n-    \"\"\"\n-    Represents an ``<input type=\"datetime-local\">``.\n-    \"\"\"\n-\n-    widget = widgets.DateTimeLocalInput()\n-\n-\n-class IntegerField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput()\n-\n-\n-class DecimalField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"number\">``.\n-    \"\"\"\n-\n-    widget = widgets.NumberInput(step=\"any\")\n-\n-\n-class IntegerRangeField(core.IntegerField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput()\n-\n-\n-class DecimalRangeField(core.DecimalField):\n-    \"\"\"\n-    Represents an ``<input type=\"range\">``.\n-    \"\"\"\n-\n-    widget = widgets.RangeInput(step=\"any\")\ndiff --git a/src/wtforms/validators.py b/src/wtforms/validators.py\nindex 2d6bea2..059f32c 100644\n--- a/src/wtforms/validators.py\n+++ b/src/wtforms/validators.py\n@@ -114,6 +114,8 @@ class Length:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)d` and `%(max)d` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `minlength` and `maxlength` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=-1, max=-1, message=None):\n@@ -124,6 +126,11 @@ class Length:\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min != -1:\n+            self.field_flags[\"minlength\"] = self.min\n+        if self.max != -1:\n+            self.field_flags[\"maxlength\"] = self.max\n \n     def __call__(self, form, field):\n         length = field.data and len(field.data) or 0\n@@ -174,12 +181,19 @@ class NumberRange:\n         Error message to raise in case of a validation error. Can be\n         interpolated using `%(min)s` and `%(max)s` if desired. Useful defaults\n         are provided depending on the existence of min and max.\n+\n+    When supported, sets the `min` and `max` attributes on widgets.\n     \"\"\"\n \n     def __init__(self, min=None, max=None, message=None):\n         self.min = min\n         self.max = max\n         self.message = message\n+        self.field_flags = {}\n+        if self.min is not None:\n+            self.field_flags[\"min\"] = self.min\n+        if self.max is not None:\n+            self.field_flags[\"max\"] = self.max\n \n     def __call__(self, form, field):\n         data = field.data\n@@ -215,9 +229,9 @@ class Optional:\n     :param strip_whitespace:\n         If True (the default) also stop the validation chain on input which\n         consists of only whitespace.\n-    \"\"\"\n \n-    field_flags = (\"optional\",)\n+    Sets the `optional` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, strip_whitespace=True):\n         if strip_whitespace:\n@@ -225,6 +239,8 @@ class Optional:\n         else:\n             self.string_check = lambda s: s\n \n+        self.field_flags = {\"optional\": True}\n+\n     def __call__(self, form, field):\n         if (\n             not field.raw_data\n@@ -256,12 +272,13 @@ class DataRequired:\n \n     :param message:\n         Error message to raise in case of a validation error.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.data or isinstance(field.data, str) and not field.data.strip():\n@@ -281,12 +298,13 @@ class InputRequired:\n     Note there is a distinction between this and DataRequired in that\n     InputRequired looks that form-input data was provided, and DataRequired\n     looks at the post-coercion data.\n-    \"\"\"\n \n-    field_flags = (\"required\",)\n+    Sets the `required` attribute on widgets.\n+    \"\"\"\n \n     def __init__(self, message=None):\n         self.message = message\n+        self.field_flags = {\"required\": True}\n \n     def __call__(self, form, field):\n         if not field.raw_data or not field.raw_data[0]:\ndiff --git a/src/wtforms/widgets/core.py b/src/wtforms/widgets/core.py\nindex 3e2a2eb..a0e0e61 100644\n--- a/src/wtforms/widgets/core.py\n+++ b/src/wtforms/widgets/core.py\n@@ -3,17 +3,30 @@ from markupsafe import Markup\n \n __all__ = (\n     \"CheckboxInput\",\n+    \"ColorInput\",\n+    \"DateInput\",\n+    \"DateTimeInput\",\n+    \"DateTimeLocalInput\",\n+    \"EmailInput\",\n     \"FileInput\",\n     \"HiddenInput\",\n     \"ListWidget\",\n+    \"MonthInput\",\n+    \"NumberInput\",\n+    \"Option\",\n     \"PasswordInput\",\n     \"RadioInput\",\n+    \"RangeInput\",\n+    \"SearchInput\",\n     \"Select\",\n     \"SubmitInput\",\n     \"TableWidget\",\n     \"TextArea\",\n     \"TextInput\",\n-    \"Option\",\n+    \"TelInput\",\n+    \"TimeInput\",\n+    \"URLInput\",\n+    \"WeekInput\",\n )\n \n \n@@ -148,6 +161,7 @@ class Input:\n     \"\"\"\n \n     html_params = staticmethod(html_params)\n+    validation_attrs = [\"required\"]\n \n     def __init__(self, input_type=None):\n         if input_type is not None:\n@@ -158,8 +172,10 @@ class Input:\n         kwargs.setdefault(\"type\", self.input_type)\n         if \"value\" not in kwargs:\n             kwargs[\"value\"] = field._value()\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\"<input %s>\" % self.html_params(name=field.name, **kwargs))\n \n \n@@ -169,6 +185,7 @@ class TextInput(Input):\n     \"\"\"\n \n     input_type = \"text\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n \n class PasswordInput(Input):\n@@ -181,6 +198,7 @@ class PasswordInput(Input):\n     \"\"\"\n \n     input_type = \"password\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n \n     def __init__(self, hide_value=True):\n         self.hide_value = hide_value\n@@ -196,9 +214,12 @@ class HiddenInput(Input):\n     Render a hidden input.\n     \"\"\"\n \n-    field_flags = (\"hidden\",)\n     input_type = \"hidden\"\n \n+    def __init__(self, *args, **kwargs):\n+        super().__init__(*args, **kwargs)\n+        self.field_flags = {\"hidden\": True}\n+\n \n class CheckboxInput(Input):\n     \"\"\"\n@@ -238,6 +259,7 @@ class FileInput(Input):\n     \"\"\"\n \n     input_type = \"file\"\n+    validation_attrs = [\"required\", \"accept\"]\n \n     def __init__(self, multiple=False):\n         super().__init__()\n@@ -275,10 +297,14 @@ class TextArea:\n     `rows` and `cols` ought to be passed as keyword args when rendering.\n     \"\"\"\n \n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\"]\n+\n     def __call__(self, field, **kwargs):\n         kwargs.setdefault(\"id\", field.id)\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         return Markup(\n             \"<textarea %s>\\r\\n%s</textarea>\"\n             % (html_params(name=field.name, **kwargs), escape(field._value()))\n@@ -297,6 +323,8 @@ class Select:\n     `(value, label, selected)`.\n     \"\"\"\n \n+    validation_attrs = [\"required\"]\n+\n     def __init__(self, multiple=False):\n         self.multiple = multiple\n \n@@ -304,8 +332,10 @@ class Select:\n         kwargs.setdefault(\"id\", field.id)\n         if self.multiple:\n             kwargs[\"multiple\"] = True\n-        if \"required\" not in kwargs and \"required\" in getattr(field, \"flags\", []):\n-            kwargs[\"required\"] = True\n+        flags = getattr(field, \"flags\", {})\n+        for k in dir(flags):\n+            if k in self.validation_attrs and k not in kwargs:\n+                kwargs[k] = getattr(flags, k)\n         html = [\"<select %s>\" % html_params(name=field.name, **kwargs)]\n         for val, label, selected in field.iter_choices():\n             html.append(self.render_option(val, label, selected))\n@@ -338,3 +368,141 @@ class Option:\n         return Select.render_option(\n             field._value(), field.label.text, field.checked, **kwargs\n         )\n+\n+\n+class SearchInput(Input):\n+    \"\"\"\n+    Renders an input with type \"search\".\n+    \"\"\"\n+\n+    input_type = \"search\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class TelInput(Input):\n+    \"\"\"\n+    Renders an input with type \"tel\".\n+    \"\"\"\n+\n+    input_type = \"tel\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class URLInput(Input):\n+    \"\"\"\n+    Renders an input with type \"url\".\n+    \"\"\"\n+\n+    input_type = \"url\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class EmailInput(Input):\n+    \"\"\"\n+    Renders an input with type \"email\".\n+    \"\"\"\n+\n+    input_type = \"email\"\n+    validation_attrs = [\"required\", \"maxlength\", \"minlength\", \"pattern\"]\n+\n+\n+class DateTimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime\".\n+    \"\"\"\n+\n+    input_type = \"datetime\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateInput(Input):\n+    \"\"\"\n+    Renders an input with type \"date\".\n+    \"\"\"\n+\n+    input_type = \"date\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class MonthInput(Input):\n+    \"\"\"\n+    Renders an input with type \"month\".\n+    \"\"\"\n+\n+    input_type = \"month\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class WeekInput(Input):\n+    \"\"\"\n+    Renders an input with type \"week\".\n+    \"\"\"\n+\n+    input_type = \"week\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class TimeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"time\".\n+    \"\"\"\n+\n+    input_type = \"time\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class DateTimeLocalInput(Input):\n+    \"\"\"\n+    Renders an input with type \"datetime-local\".\n+    \"\"\"\n+\n+    input_type = \"datetime-local\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+\n+class NumberInput(Input):\n+    \"\"\"\n+    Renders an input with type \"number\".\n+    \"\"\"\n+\n+    input_type = \"number\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None, min=None, max=None):\n+        self.step = step\n+        self.min = min\n+        self.max = max\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        if self.min is not None:\n+            kwargs.setdefault(\"min\", self.min)\n+        if self.max is not None:\n+            kwargs.setdefault(\"max\", self.max)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class RangeInput(Input):\n+    \"\"\"\n+    Renders an input with type \"range\".\n+    \"\"\"\n+\n+    input_type = \"range\"\n+    validation_attrs = [\"required\", \"max\", \"min\", \"step\"]\n+\n+    def __init__(self, step=None):\n+        self.step = step\n+\n+    def __call__(self, field, **kwargs):\n+        if self.step is not None:\n+            kwargs.setdefault(\"step\", self.step)\n+        return super().__call__(field, **kwargs)\n+\n+\n+class ColorInput(Input):\n+    \"\"\"\n+    Renders an input with type \"color\".\n+    \"\"\"\n+\n+    input_type = \"color\"\ndiff --git a/src/wtforms/widgets/html5.py b/src/wtforms/widgets/html5.py\ndeleted file mode 100644\nindex 2a9ea76..0000000\n--- a/src/wtforms/widgets/html5.py\n+++ /dev/null\n@@ -1,143 +0,0 @@\n-from .core import Input\n-\n-__all__ = (\n-    \"ColorInput\",\n-    \"DateInput\",\n-    \"DateTimeInput\",\n-    \"DateTimeLocalInput\",\n-    \"EmailInput\",\n-    \"MonthInput\",\n-    \"NumberInput\",\n-    \"RangeInput\",\n-    \"SearchInput\",\n-    \"TelInput\",\n-    \"TimeInput\",\n-    \"URLInput\",\n-    \"WeekInput\",\n-)\n-\n-\n-class SearchInput(Input):\n-    \"\"\"\n-    Renders an input with type \"search\".\n-    \"\"\"\n-\n-    input_type = \"search\"\n-\n-\n-class TelInput(Input):\n-    \"\"\"\n-    Renders an input with type \"tel\".\n-    \"\"\"\n-\n-    input_type = \"tel\"\n-\n-\n-class URLInput(Input):\n-    \"\"\"\n-    Renders an input with type \"url\".\n-    \"\"\"\n-\n-    input_type = \"url\"\n-\n-\n-class EmailInput(Input):\n-    \"\"\"\n-    Renders an input with type \"email\".\n-    \"\"\"\n-\n-    input_type = \"email\"\n-\n-\n-class DateTimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime\".\n-    \"\"\"\n-\n-    input_type = \"datetime\"\n-\n-\n-class DateInput(Input):\n-    \"\"\"\n-    Renders an input with type \"date\".\n-    \"\"\"\n-\n-    input_type = \"date\"\n-\n-\n-class MonthInput(Input):\n-    \"\"\"\n-    Renders an input with type \"month\".\n-    \"\"\"\n-\n-    input_type = \"month\"\n-\n-\n-class WeekInput(Input):\n-    \"\"\"\n-    Renders an input with type \"week\".\n-    \"\"\"\n-\n-    input_type = \"week\"\n-\n-\n-class TimeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"time\".\n-    \"\"\"\n-\n-    input_type = \"time\"\n-\n-\n-class DateTimeLocalInput(Input):\n-    \"\"\"\n-    Renders an input with type \"datetime-local\".\n-    \"\"\"\n-\n-    input_type = \"datetime-local\"\n-\n-\n-class NumberInput(Input):\n-    \"\"\"\n-    Renders an input with type \"number\".\n-    \"\"\"\n-\n-    input_type = \"number\"\n-\n-    def __init__(self, step=None, min=None, max=None):\n-        self.step = step\n-        self.min = min\n-        self.max = max\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        if self.min is not None:\n-            kwargs.setdefault(\"min\", self.min)\n-        if self.max is not None:\n-            kwargs.setdefault(\"max\", self.max)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class RangeInput(Input):\n-    \"\"\"\n-    Renders an input with type \"range\".\n-    \"\"\"\n-\n-    input_type = \"range\"\n-\n-    def __init__(self, step=None):\n-        self.step = step\n-\n-    def __call__(self, field, **kwargs):\n-        if self.step is not None:\n-            kwargs.setdefault(\"step\", self.step)\n-        return super().__call__(field, **kwargs)\n-\n-\n-class ColorInput(Input):\n-    \"\"\"\n-    Renders an input with type \"color\".\n-    \"\"\"\n-\n-    input_type = \"color\"\n",
        "tests": {
          "test_patch": "diff --git a/tests/test_fields.py b/tests/test_fields.py\nindex f36ce71..0986534 100644\n--- a/tests/test_fields.py\n+++ b/tests/test_fields.py\n@@ -119,7 +119,7 @@ class TestFlags:\n     def test_existing_values(self, flags):\n         assert flags.required is True\n         assert \"required\" in flags\n-        assert flags.optional is False\n+        assert flags.optional is None\n         assert \"optional\" not in flags\n \n     def test_assignment(self, flags):\n@@ -1373,3 +1373,111 @@ class TestHTML5Fields:\n             if field.data != item[\"data\"]:\n                 tmpl = \"Field {key} data mismatch: {field.data!r} != {data!r}\"\n                 raise AssertionError(tmpl.format(field=field, **item))\n+\n+\n+class TestFieldValidatorsTest:\n+    class F(Form):\n+        v1 = [validators.Length(min=1)]\n+        v2 = [validators.Length(min=1, max=3)]\n+        string1 = StringField(validators=v1)\n+        string2 = StringField(validators=v2)\n+        password1 = PasswordField(validators=v1)\n+        password2 = PasswordField(validators=v2)\n+        textarea1 = TextAreaField(validators=v1)\n+        textarea2 = TextAreaField(validators=v2)\n+        search1 = SearchField(validators=v1)\n+        search2 = SearchField(validators=v2)\n+\n+        v3 = [validators.NumberRange(min=1)]\n+        v4 = [validators.NumberRange(min=1, max=3)]\n+        integer1 = IntegerField(validators=v3)\n+        integer2 = IntegerField(validators=v4)\n+        integerrange1 = IntegerRangeField(validators=v3)\n+        integerrange2 = IntegerRangeField(validators=v4)\n+        decimal1 = DecimalField(validators=v3)\n+        decimal2 = DecimalField(validators=v4)\n+        decimalrange1 = DecimalRangeField(validators=v3)\n+        decimalrange2 = DecimalRangeField(validators=v4)\n+\n+    def test_minlength_maxlength(self):\n+        form = self.F()\n+        assert (\n+            form.string1()\n+            == '<input id=\"string1\" minlength=\"1\" name=\"string1\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.string2() == '<input id=\"string2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"string2\" type=\"text\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password1() == '<input id=\"password1\" minlength=\"1\"'\n+            ' name=\"password1\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.password2() == '<input id=\"password2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"password2\" type=\"password\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.textarea1() == '<textarea id=\"textarea1\" minlength=\"1\"'\n+            ' name=\"textarea1\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.textarea2() == '<textarea id=\"textarea2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"textarea2\">\\r\\n</textarea>'\n+        )\n+\n+        assert (\n+            form.search1() == '<input id=\"search1\" minlength=\"1\"'\n+            ' name=\"search1\" type=\"search\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.search2() == '<input id=\"search2\" maxlength=\"3\" minlength=\"1\"'\n+            ' name=\"search2\" type=\"search\" value=\"\">'\n+        )\n+\n+    def test_min_max(self):\n+        form = self.F()\n+        assert (\n+            form.integer1()\n+            == '<input id=\"integer1\" min=\"1\" name=\"integer1\" type=\"number\" value=\"\">'\n+        )\n+        assert (\n+            form.integer2() == '<input id=\"integer2\" max=\"3\" min=\"1\"'\n+            ' name=\"integer2\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange1() == '<input id=\"integerrange1\" min=\"1\"'\n+            ' name=\"integerrange1\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.integerrange2() == '<input id=\"integerrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"integerrange2\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal1() == '<input id=\"decimal1\" min=\"1\"'\n+            ' name=\"decimal1\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimal2() == '<input id=\"decimal2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimal2\" step=\"any\" type=\"number\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange1() == '<input id=\"decimalrange1\" min=\"1\"'\n+            ' name=\"decimalrange1\" step=\"any\" type=\"range\" value=\"\">'\n+        )\n+\n+        assert (\n+            form.decimalrange2() == '<input id=\"decimalrange2\" max=\"3\" min=\"1\"'\n+            ' name=\"decimalrange2\" step=\"any\" type=\"range\" value=\"\">'\n+        )\ndiff --git a/tests/test_validators.py b/tests/test_validators.py\nindex dd4613c..4e373ff 100644\n--- a/tests/test_validators.py\n+++ b/tests/test_validators.py\n@@ -31,7 +31,7 @@ def test_data_required(dummy_form, dummy_field):\n     validator = data_required()\n     dummy_field.data = \"foobar\"\n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n @pytest.mark.parametrize(\"bad_val\", [None, \"\", \" \", \"\\t\\t\"])\n@@ -85,7 +85,7 @@ def test_input_required(dummy_form, dummy_field):\n     dummy_field.raw_data = [\"foobar\"]\n \n     validator(dummy_form, dummy_field)\n-    assert validator.field_flags == (\"required\",)\n+    assert validator.field_flags == {\"required\": True}\n \n \n def test_input_required_raises(dummy_form, dummy_field):\n@@ -141,7 +141,7 @@ def test_input_optional_raises(data_v, raw_data_v, dummy_form, dummy_field):\n \n     with pytest.raises(StopValidation):\n         validator(dummy_form, dummy_field)\n-        assert validator.field_flags == (\"optional\",)\n+        assert validator.field_flags == {\"optional\": True}\n \n     dummy_field.errors = [\"Invalid Integer Value\"]\n     assert len(dummy_field.errors) == 1\ndiff --git a/tests/test_widgets.py b/tests/test_widgets.py\nindex 9bedec6..a9b7cd3 100644\n--- a/tests/test_widgets.py\n+++ b/tests/test_widgets.py\n@@ -7,14 +7,14 @@ from wtforms.widgets.core import HiddenInput\n from wtforms.widgets.core import html_params\n from wtforms.widgets.core import Input\n from wtforms.widgets.core import ListWidget\n+from wtforms.widgets.core import NumberInput\n from wtforms.widgets.core import PasswordInput\n from wtforms.widgets.core import RadioInput\n+from wtforms.widgets.core import RangeInput\n from wtforms.widgets.core import Select\n from wtforms.widgets.core import TableWidget\n from wtforms.widgets.core import TextArea\n from wtforms.widgets.core import TextInput\n-from wtforms.widgets.html5 import NumberInput\n-from wtforms.widgets.html5 import RangeInput\n \n \n class TestHTMLParams:\n",
          "FAIL_TO_PASS": [
            "tests/test_fields.py::TestDefaults::test",
            "tests/test_fields.py::TestLabel::test",
            "tests/test_fields.py::TestLabel::test_auto_label",
            "tests/test_fields.py::TestLabel::test_override_for",
            "tests/test_fields.py::TestLabel::test_escaped_label_text",
            "tests/test_fields.py::TestFlags::test_existing_values",
            "tests/test_fields.py::TestFlags::test_assignment",
            "tests/test_fields.py::TestFlags::test_unset",
            "tests/test_fields.py::TestFlags::test_repr",
            "tests/test_fields.py::TestFlags::test_underscore_property",
            "tests/test_fields.py::TestUnsetValue::test",
            "tests/test_fields.py::TestFilters::test_working",
            "tests/test_fields.py::TestFilters::test_failure",
            "tests/test_fields.py::TestFilters::test_extra",
            "tests/test_fields.py::TestFilters::test_inline",
            "tests/test_fields.py::TestField::test_unbound_field",
            "tests/test_fields.py::TestField::test_htmlstring",
            "tests/test_fields.py::TestField::test_str_coerce",
            "tests/test_fields.py::TestField::test_unicode_coerce",
            "tests/test_fields.py::TestField::test_process_formdata",
            "tests/test_fields.py::TestField::test_meta_attribute",
            "tests/test_fields.py::TestField::test_render_kw",
            "tests/test_fields.py::TestField::test_render_special",
            "tests/test_fields.py::TestField::test_select_field_copies_choices",
            "tests/test_fields.py::TestField::test_required_flag",
            "tests/test_fields.py::TestField::test_check_validators",
            "tests/test_fields.py::TestField::test_custom_name",
            "tests/test_fields.py::TestPrePostValidation::test_pre_stop",
            "tests/test_fields.py::TestPrePostValidation::test_post",
            "tests/test_fields.py::TestSelectField::test_defaults",
            "tests/test_fields.py::TestSelectField::test_with_data",
            "tests/test_fields.py::TestSelectField::test_value_coercion",
            "tests/test_fields.py::TestSelectField::test_iterable_options",
            "tests/test_fields.py::TestSelectField::test_default_coerce",
            "tests/test_fields.py::TestSelectField::test_validate_choices",
            "tests/test_fields.py::TestSelectField::test_validate_choices_when_empty",
            "tests/test_fields.py::TestSelectField::test_validate_choices_when_none",
            "tests/test_fields.py::TestSelectField::test_dont_validate_choices",
            "tests/test_fields.py::TestSelectField::test_choice_shortcut",
            "tests/test_fields.py::TestSelectField::test_empty_choice[choices0]",
            "tests/test_fields.py::TestSelectField::test_empty_choice[None]",
            "tests/test_fields.py::TestSelectField::test_callable_choices",
            "tests/test_fields.py::TestSelectMultipleField::test_defaults",
            "tests/test_fields.py::TestSelectMultipleField::test_with_data",
            "tests/test_fields.py::TestSelectMultipleField::test_coerce_fail",
            "tests/test_fields.py::TestSelectMultipleField::test_callable_choices",
            "tests/test_fields.py::TestSelectMultipleField::test_choice_shortcut",
            "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[choices0]",
            "tests/test_fields.py::TestSelectMultipleField::test_empty_choice[None]",
            "tests/test_fields.py::TestRadioField::test",
            "tests/test_fields.py::TestRadioField::test_text_coercion",
            "tests/test_fields.py::TestRadioField::test_callable_choices",
            "tests/test_fields.py::TestStringField::test",
            "tests/test_fields.py::TestHiddenField::test",
            "tests/test_fields.py::TestTextAreaField::test",
            "tests/test_fields.py::TestPasswordField::test",
            "tests/test_fields.py::TestFileField::test_file_field",
            "tests/test_fields.py::TestFileField::test_multiple_file_field",
            "tests/test_fields.py::TestFileField::test_file_field_without_file_input",
            "tests/test_fields.py::TestIntegerField::test",
            "tests/test_fields.py::TestDecimalField::test",
            "tests/test_fields.py::TestDecimalField::test_quantize",
            "tests/test_fields.py::TestFloatField::test",
            "tests/test_fields.py::TestBooleanField::test_defaults",
            "tests/test_fields.py::TestBooleanField::test_rendering",
            "tests/test_fields.py::TestBooleanField::test_with_postdata",
            "tests/test_fields.py::TestBooleanField::test_with_model_data",
            "tests/test_fields.py::TestBooleanField::test_with_postdata_and_model",
            "tests/test_fields.py::TestDateField::test_basic",
            "tests/test_fields.py::TestDateField::test_failure",
            "tests/test_fields.py::TestMonthField::test_basic",
            "tests/test_fields.py::TestMonthField::test_failure",
            "tests/test_fields.py::TestTimeField::test_basic",
            "tests/test_fields.py::TestDateTimeField::test_basic",
            "tests/test_fields.py::TestDateTimeField::test_microseconds",
            "tests/test_fields.py::TestSubmitField::test",
            "tests/test_fields.py::TestFormField::test_formdata",
            "tests/test_fields.py::TestFormField::test_iteration",
            "tests/test_fields.py::TestFormField::test_with_obj",
            "tests/test_fields.py::TestFormField::test_widget",
            "tests/test_fields.py::TestFormField::test_separator",
            "tests/test_fields.py::TestFormField::test_no_validators_or_filters",
            "tests/test_fields.py::TestFormField::test_populate_missing_obj",
            "tests/test_fields.py::TestFieldList::test_form",
            "tests/test_fields.py::TestFieldList::test_enclosed_subform",
            "tests/test_fields.py::TestFieldList::test_enclosed_subform_custom_name",
            "tests/test_fields.py::TestFieldList::test_entry_management",
            "tests/test_fields.py::TestFieldList::test_min_max_entries",
            "tests/test_fields.py::TestFieldList::test_validators",
            "tests/test_fields.py::TestFieldList::test_no_filters",
            "tests/test_fields.py::TestFieldList::test_process_prefilled",
            "tests/test_fields.py::TestFieldList::test_errors",
            "tests/test_fields.py::TestCustomFieldQuirks::test_processing_failure",
            "tests/test_fields.py::TestCustomFieldQuirks::test_default_impls",
            "tests/test_fields.py::TestHTML5Fields::test_simple",
            "tests/test_fields.py::TestFieldValidatorsTest::test_minlength_maxlength",
            "tests/test_fields.py::TestFieldValidatorsTest::test_min_max",
            "tests/test_validators.py::test_data_required",
            "tests/test_validators.py::test_data_required_raises[None]",
            "tests/test_validators.py::test_data_required_raises[]",
            "tests/test_validators.py::test_data_required_raises[",
            "tests/test_validators.py::test_data_required_raises[\\t\\t]",
            "tests/test_validators.py::test_data_required_clobber",
            "tests/test_validators.py::test_data_required_messages[validator0-This",
            "tests/test_validators.py::test_data_required_messages[validator1-foo]",
            "tests/test_validators.py::test_input_required",
            "tests/test_validators.py::test_input_required_raises",
            "tests/test_validators.py::test_input_required_error_message[validator0-This",
            "tests/test_validators.py::test_input_required_error_message[validator1-foo]",
            "tests/test_validators.py::test_input_optional_passes",
            "tests/test_validators.py::test_input_optional_raises[-]",
            "tests/test_validators.py::test_input_optional_raises[",
            "tests/test_validators.py::test_input_optional_raises[\\t-\\t]",
            "tests/test_validators.py::test_ip4address_passes[147.230.23.25]",
            "tests/test_validators.py::test_ip4address_passes[147.230.23.0]",
            "tests/test_validators.py::test_ip4address_passes[127.0.0.1]",
            "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::1111]",
            "tests/test_validators.py::test_good_ip6address_passes[2001:718:1C01:1111::]",
            "tests/test_validators.py::test_good_ip6address_passes[::1]",
            "tests/test_validators.py::test_good_ip6address_passes[dead:beef:0:0:0:0:42:1]",
            "tests/test_validators.py::test_good_ip6address_passes[abcd:ef::42:1]",
            "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::1111]",
            "tests/test_validators.py::test_bad_ip6address_raises[2001:718:1C01:1111::]",
            "tests/test_validators.py::test_bad_ip6address_raises[abc.0.0.1]",
            "tests/test_validators.py::test_bad_ip6address_raises[abcd:1234::123::1]",
            "tests/test_validators.py::test_bad_ip6address_raises[1:2:3:4:5:6:7:8:9]",
            "tests/test_validators.py::test_bad_ip6address_raises[abcd::1ffff]",
            "tests/test_validators.py::test_bad_ip4address_raises[147.230.1000.25]",
            "tests/test_validators.py::test_bad_ip4address_raises[2001:718::::]",
            "tests/test_validators.py::test_bad_ip4address_raises[abc.0.0.1]",
            "tests/test_validators.py::test_bad_ip4address_raises[1278.0.0.1]",
            "tests/test_validators.py::test_bad_ip4address_raises[127.0.0.abc]",
            "tests/test_validators.py::test_bad_ip4address_raises[900.200.100.75]",
            "tests/test_validators.py::test_valid_email_passes[foo@bar.dk]",
            "tests/test_validators.py::test_valid_email_passes[123@bar.dk]",
            "tests/test_validators.py::test_valid_email_passes[foo@456.dk]",
            "tests/test_validators.py::test_valid_email_passes[foo@bar456.info]",
            "tests/test_validators.py::test_valid_email_passes[foo@b\\xfccher.\\u4e2d\\u56fd]",
            "tests/test_validators.py::test_invalid_email_raises[None]",
            "tests/test_validators.py::test_invalid_email_raises[]",
            "tests/test_validators.py::test_invalid_email_raises[",
            "tests/test_validators.py::test_invalid_email_raises[foo]",
            "tests/test_validators.py::test_invalid_email_raises[bar.dk]",
            "tests/test_validators.py::test_invalid_email_raises[foo@]",
            "tests/test_validators.py::test_invalid_email_raises[@bar.dk]",
            "tests/test_validators.py::test_invalid_email_raises[foo@bar]",
            "tests/test_validators.py::test_invalid_email_raises[foo@bar.ab12]",
            "tests/test_validators.py::test_invalid_email_raises[foo@.bar.ab]",
            "tests/test_validators.py::test_invalid_email_raises[foo.@bar.co]",
            "tests/test_validators.py::test_invalid_email_raises[foo@foo@bar.co]",
            "tests/test_validators.py::test_invalid_email_raises[fo",
            "tests/test_validators.py::test_invalid_email_raises_granular_message[foo@]",
            "tests/test_validators.py::test_ip_address_raises_on_bad_init",
            "tests/test_validators.py::test_valid_mac_passes[01:23:45:67:ab:CD]",
            "tests/test_validators.py::test_bad_mac_raises[00:00:00:00:00]",
            "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:]",
            "tests/test_validators.py::test_bad_mac_raises[01:23:45:67:89:gh]",
            "tests/test_validators.py::test_bad_mac_raises[123:23:45:67:89:00]",
            "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f-0deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_valid_uuid_passes[2bc1c94f0deb43e992a14775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f-0deb-43e9-92a1-4775189ec9f]",
            "tests/test_validators.py::test_bad_uuid_raises[gbc1c94f-0deb-43e9-92a1-4775189ec9f8]",
            "tests/test_validators.py::test_bad_uuid_raises[2bc1c94f",
            "tests/test_validators.py::test_required_passes",
            "tests/test_validators.py::test_required_stops_validation",
            "tests/test_validators.py::test_equal_to_passes",
            "tests/test_validators.py::test_equal_to_raises[test-invalid_field_name]",
            "tests/test_validators.py::test_equal_to_raises[bad_value-foo]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo-bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo--bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foo_bar.dk/]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk?query=param]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.dk/path?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_passes[http://foobar.museum/foobar]",
            "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1/foobar]",
            "tests/test_validators.py::test_valid_url_passes[http://192.168.0.1:9000/fake]",
            "tests/test_validators.py::test_valid_url_passes[http://\\u0645\\u062b\\u0627\\u0644.\\u0625\\u062e\\u062a\\u0628\\u0627\\u0631/foo.com]",
            "tests/test_validators.py::test_valid_url_passes[http://\\u0909\\u0926\\u093e\\u0939\\u0930\\u0923.\\u092a\\u0930\\u0940\\u0915\\u094d\\u0937\\u093e/]",
            "tests/test_validators.py::test_valid_url_passes[http://\\uc2e4\\ub840.\\ud14c\\uc2a4\\ud2b8]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://localhost/foobar]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:5000?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar/path?query=param&foo=faa]",
            "tests/test_validators.py::test_valid_url_notld_passes[http://foobar:1234/path?query=param&foo=faa]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar]",
            "tests/test_validators.py::test_bad_url_raises[http://-foobar.dk/]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar-.dk/]",
            "tests/test_validators.py::test_bad_url_raises[foobar.dk]",
            "tests/test_validators.py::test_bad_url_raises[http://127.0.0/asdf]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar.d]",
            "tests/test_validators.py::test_bad_url_raises[http://foobar.12]",
            "tests/test_validators.py::test_bad_url_raises[http://localhost:abc/a]",
            "tests/test_validators.py::test_anyof_passes[b-test_list0]",
            "tests/test_validators.py::test_anyof_passes[2-test_list1]",
            "tests/test_validators.py::test_anyof_raisses[d-test_list0]",
            "tests/test_validators.py::test_anyof_raisses[6-test_list1]",
            "tests/test_validators.py::test_any_of_values_formatter",
            "tests/test_validators.py::test_none_of_passes",
            "tests/test_validators.py::test_none_of_raises",
            "tests/test_validators.py::test_correct_length_passes[2-6]",
            "tests/test_validators.py::test_correct_length_passes[6--1]",
            "tests/test_validators.py::test_correct_length_passes[-1-6]",
            "tests/test_validators.py::test_correct_length_passes[6-6]",
            "tests/test_validators.py::test_bad_length_raises[7--1]",
            "tests/test_validators.py::test_bad_length_raises[-1-5]",
            "tests/test_validators.py::test_bad_length_init_raises[-1--1]",
            "tests/test_validators.py::test_bad_length_init_raises[5-2]",
            "tests/test_validators.py::test_length_messages[validator0-2",
            "tests/test_validators.py::test_length_messages[validator1-at",
            "tests/test_validators.py::test_length_messages[validator2-longer",
            "tests/test_validators.py::test_length_messages[validator3-between",
            "tests/test_validators.py::test_length_messages[validator4-exactly",
            "tests/test_validators.py::test_number_range_passes[5-10-7]",
            "tests/test_validators.py::test_number_range_passes[5-None-7]",
            "tests/test_validators.py::test_number_range_passes[None-100-70]",
            "tests/test_validators.py::test_number_range_raises[5-10-None]",
            "tests/test_validators.py::test_number_range_raises[5-10-0]",
            "tests/test_validators.py::test_number_range_raises[5-10-12]",
            "tests/test_validators.py::test_number_range_raises[5-10--5]",
            "tests/test_validators.py::test_number_range_raises[5-None-4]",
            "tests/test_validators.py::test_number_range_raises[None-100-500]",
            "tests/test_validators.py::test_number_range_nan[nan]",
            "tests/test_validators.py::test_number_range_nan[nan1]",
            "tests/test_validators.py::test_lazy_proxy_raises",
            "tests/test_validators.py::test_lazy_proxy_fixture",
            "tests/test_validators.py::test_regex_passes[^a-None-abcd-a0]",
            "tests/test_validators.py::test_regex_passes[^a-re.IGNORECASE-ABcd-A]",
            "tests/test_validators.py::test_regex_passes[^a-None-abcd-a1]",
            "tests/test_validators.py::test_regex_passes[^a-None-ABcd-A]",
            "tests/test_validators.py::test_regex_raises[^a-None-ABC]",
            "tests/test_validators.py::test_regex_raises[^a-re.IGNORECASE-foo]",
            "tests/test_validators.py::test_regex_raises[^a-None-None0]",
            "tests/test_validators.py::test_regex_raises[^a-None-foo]",
            "tests/test_validators.py::test_regex_raises[^a-None-None1]",
            "tests/test_validators.py::test_regexp_message",
            "tests/test_validators.py::test_raise_exceptions[IndexError]",
            "tests/test_validators.py::test_raise_exceptions[ZeroDivisionError]",
            "tests/test_validators.py::test_raise_exceptions[ValueError]",
            "tests/test_widgets.py::TestHTMLParams::test_basic",
            "tests/test_widgets.py::TestHTMLParams::test_data_prefix",
            "tests/test_widgets.py::TestHTMLParams::test_aria_prefix",
            "tests/test_widgets.py::TestHTMLParams::test_quoting",
            "tests/test_widgets.py::TestListWidget::test_listwidget",
            "tests/test_widgets.py::TestTableWidget::test_tablewidget",
            "tests/test_widgets.py::TestBasicWidgets::test_input_type",
            "tests/test_widgets.py::TestBasicWidgets::test_html_marking",
            "tests/test_widgets.py::TestBasicWidgets::test_text_input",
            "tests/test_widgets.py::TestBasicWidgets::test_password_input",
            "tests/test_widgets.py::TestBasicWidgets::test_hidden_input",
            "tests/test_widgets.py::TestBasicWidgets::test_checkbox_input",
            "tests/test_widgets.py::TestBasicWidgets::test_radio_input",
            "tests/test_widgets.py::TestBasicWidgets::test_textarea",
            "tests/test_widgets.py::TestBasicWidgets::test_file",
            "tests/test_widgets.py::TestSelect::test_select",
            "tests/test_widgets.py::TestSelect::test_render_option",
            "tests/test_widgets.py::TestHTML5::test_number",
            "tests/test_widgets.py::TestHTML5::test_range"
          ],
          "PASS_TO_PASS": []
        },
        "warning": "Repository tests and the gold patch have not been executed here. Response probes are implementation sketches."
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "verified:mbpp:repair:602",
      "title": "Repair first-repeated-character detection",
      "source": {
        "dataset": "google-research-datasets/mbpp",
        "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
        "license": "CC-BY-4.0",
        "url": "https://huggingface.co/datasets/google-research-datasets/mbpp/blob/4bb6404fdc6cacfda99d4ac4205087b89d32030c/sanitized/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "602",
        "pool": "proposed",
        "file_sha256": "d95f8ad6d2fff08fe4826122d6e3e31f75716825d0c5c340d297aca5e9e0de0e",
        "row_sha256": "9e3c711eafa57da7e9952698fe81f4655372f5d6f3a33646aaf2de00113f99b3",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Jacob Austin et al., Program Synthesis with Large Language Models (2021), Google Research MBPP. Adapted into repair prompts; rubrics, alternative solutions and supplemental tests added for this gallery."
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "software_repair",
        "leaf": "software_repair.functional_bug_repair",
        "confidence": "high",
        "reason": "Explicitly adapted from a function-writing task into a repair task; original prompt retained below."
      },
      "language": "en",
      "prompt": "Repair this Python implementation. Scan left to right and return the character whose second occurrence is encountered first, or None if no character repeats. Character identity is case-sensitive. Keep the first_repeated_char(s) call signature.\n\n```python\ndef first_repeated_char(s):\n    return next((c for c in s if s.count(c) > 1), None)\n```",
      "original_prompt": "Write a python function to find the first repeated character in a given string.",
      "context": [
        "Original MBPP prompt: Write a python function to find the first repeated character in a given string.",
        "Task adaptation: a faulty implementation and explicit callable/behavior contract were added for this gallery. Grounding distinguishes source tests from supplemental checks."
      ],
      "environment_note": "Self-contained Python. The displayed implementations were executed locally; no external repository is required.",
      "rubric": {
        "markdown": "# Repair first-repeated-character detection\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct behavior on source cases\n\n**ID:** source\\_behavior\n\n**Criterion:** Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** All source examples pass with the requested callable.\n- **0:** One or more source examples fail or the callable cannot be invoked.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_tests\", \"adapted\\_task\\_contract\"\\]\n\n### Generalizes across the stated contract\n\n**ID:** general\\_behavior\n\n**Criterion:** Returns the first character to repeat during a left-to-right scan, or None. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** All supplemental contract checks pass.\n- **0:** A supplemental case violates the contract.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"supplemental\\_tests\", \"adapted\\_task\\_contract\"\\]\n",
        "raw": {
          "title": "Repair first-repeated-character detection",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "source_behavior",
              "criterion": "Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.",
              "weight": 3.0,
              "grading_anchors": {
                "1": "All source examples pass with the requested callable.",
                "0": "One or more source examples fail or the callable cannot be invoked."
              },
              "evidence_refs": [
                "source_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            },
            {
              "id": "general_behavior",
              "criterion": "Returns the first character to repeat during a left-to-right scan, or None. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "All supplemental contract checks pass.",
                "0": "A supplemental case violates the contract."
              },
              "evidence_refs": [
                "supplemental_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "source_behavior": {
            "title": "Correct behavior on source cases",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "general_behavior": {
            "title": "Generalizes across the stated contract",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "Function behavior defined by the explicit contract and tests.",
        "solution": "def first_repeated_char(str1):\n  for index,c in enumerate(str1):\n    if str1[:index+1].count(c) > 1:\n      return c",
        "tests": [
          "assert first_repeated_char(\"abcabc\") == \"a\"",
          "assert first_repeated_char(\"abc\") == None",
          "assert first_repeated_char(\"123123\") == \"1\""
        ],
        "source_reference_verified": true,
        "supplemental_test_count": 366,
        "warning": "The wrong implementation passes all three source tests but fails on 'abba': it returns 'a' instead of 'b'."
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Two different valid implementations pass the same checks.",
          "Incorrect controls expose failure modes and sparse-test blind spots.",
          "The wrong implementation passes all three source tests but fails on 'abba': it returns 'a' instead of 'b'."
        ],
        "risks": [
          "Finite tests do not prove correctness for every input.",
          "The explicit repair contract and supplemental tests are authored additions, not original source annotations."
        ],
        "must_hold": [
          "The second-occurrence order determines the result, not the first-occurrence order.",
          "No-repeat inputs return None; case and character identity are preserved."
        ],
        "may_vary": [
          "Set, dictionary, nested scan, or another behaviorally correct implementation.",
          "No linear-time requirement is stated; the slower prefix-search implementation is valid."
        ],
        "proposed_changes": [
          "Keep source reference code as one audited implementation, never as the required method.",
          "Expand behavior checks before scaling admission; use property-based and adversarial tests."
        ]
      },
      "probes": [
        {
          "label": "Seen-set scan",
          "kind": "valid_alternative",
          "response": "```python\ndef first_repeated_char(s):\n    seen = set()\n    for c in s:\n        if c in seen:\n            return c\n        seen.add(c)\n    return None\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 366,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert first_repeated_char(\"abcabc\") == \"a\""
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert first_repeated_char(\"abc\") == None"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert first_repeated_char(\"123123\") == \"1\""
              },
              {
                "name": "Short-string enumeration ''",
                "passed": true,
                "detail": "assert first_repeated_char('') == None"
              },
              {
                "name": "Short-string enumeration 'a'",
                "passed": true,
                "detail": "assert first_repeated_char('a') == None"
              },
              {
                "name": "Short-string enumeration 'b'",
                "passed": true,
                "detail": "assert first_repeated_char('b') == None"
              },
              {
                "name": "Short-string enumeration 'c'",
                "passed": true,
                "detail": "assert first_repeated_char('c') == None"
              },
              {
                "name": "Short-string enumeration 'aa'",
                "passed": true,
                "detail": "assert first_repeated_char('aa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ab'",
                "passed": true,
                "detail": "assert first_repeated_char('ab') == None"
              },
              {
                "name": "Short-string enumeration 'ac'",
                "passed": true,
                "detail": "assert first_repeated_char('ac') == None"
              },
              {
                "name": "Short-string enumeration 'ba'",
                "passed": true,
                "detail": "assert first_repeated_char('ba') == None"
              },
              {
                "name": "Short-string enumeration 'bb'",
                "passed": true,
                "detail": "assert first_repeated_char('bb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bc'",
                "passed": true,
                "detail": "assert first_repeated_char('bc') == None"
              },
              {
                "name": "Short-string enumeration 'ca'",
                "passed": true,
                "detail": "assert first_repeated_char('ca') == None"
              },
              {
                "name": "Short-string enumeration 'cb'",
                "passed": true,
                "detail": "assert first_repeated_char('cb') == None"
              },
              {
                "name": "Short-string enumeration 'cc'",
                "passed": true,
                "detail": "assert first_repeated_char('cc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aab'",
                "passed": true,
                "detail": "assert first_repeated_char('aab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aac'",
                "passed": true,
                "detail": "assert first_repeated_char('aac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aba'",
                "passed": true,
                "detail": "assert first_repeated_char('aba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abb'",
                "passed": true,
                "detail": "assert first_repeated_char('abb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abc'",
                "passed": true,
                "detail": "assert first_repeated_char('abc') == None"
              },
              {
                "name": "Short-string enumeration 'aca'",
                "passed": true,
                "detail": "assert first_repeated_char('aca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acb'",
                "passed": true,
                "detail": "assert first_repeated_char('acb') == None"
              },
              {
                "name": "Short-string enumeration 'acc'",
                "passed": true,
                "detail": "assert first_repeated_char('acc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baa'",
                "passed": true,
                "detail": "assert first_repeated_char('baa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bab'",
                "passed": true,
                "detail": "assert first_repeated_char('bab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bac'",
                "passed": true,
                "detail": "assert first_repeated_char('bac') == None"
              },
              {
                "name": "Short-string enumeration 'bba'",
                "passed": true,
                "detail": "assert first_repeated_char('bba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bca'",
                "passed": true,
                "detail": "assert first_repeated_char('bca') == None"
              },
              {
                "name": "Short-string enumeration 'bcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caa'",
                "passed": true,
                "detail": "assert first_repeated_char('caa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cab'",
                "passed": true,
                "detail": "assert first_repeated_char('cab') == None"
              },
              {
                "name": "Short-string enumeration 'cac'",
                "passed": true,
                "detail": "assert first_repeated_char('cac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cba'",
                "passed": true,
                "detail": "assert first_repeated_char('cba') == None"
              },
              {
                "name": "Short-string enumeration 'cbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cca'",
                "passed": true,
                "detail": "assert first_repeated_char('cca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abab'",
                "passed": true,
                "detail": "assert first_repeated_char('abab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abac'",
                "passed": true,
                "detail": "assert first_repeated_char('abac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abba'",
                "passed": true,
                "detail": "assert first_repeated_char('abba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abca'",
                "passed": true,
                "detail": "assert first_repeated_char('abca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acab'",
                "passed": true,
                "detail": "assert first_repeated_char('acab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acac'",
                "passed": true,
                "detail": "assert first_repeated_char('acac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acba'",
                "passed": true,
                "detail": "assert first_repeated_char('acba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acca'",
                "passed": true,
                "detail": "assert first_repeated_char('acca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accb'",
                "passed": true,
                "detail": "assert first_repeated_char('accb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accc'",
                "passed": true,
                "detail": "assert first_repeated_char('accc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baab'",
                "passed": true,
                "detail": "assert first_repeated_char('baab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baac'",
                "passed": true,
                "detail": "assert first_repeated_char('baac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baba'",
                "passed": true,
                "detail": "assert first_repeated_char('baba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babb'",
                "passed": true,
                "detail": "assert first_repeated_char('babb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babc'",
                "passed": true,
                "detail": "assert first_repeated_char('babc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'baca'",
                "passed": true,
                "detail": "assert first_repeated_char('baca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caab'",
                "passed": true,
                "detail": "assert first_repeated_char('caab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caac'",
                "passed": true,
                "detail": "assert first_repeated_char('caac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caba'",
                "passed": true,
                "detail": "assert first_repeated_char('caba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caca'",
                "passed": true,
                "detail": "assert first_repeated_char('caca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabab'",
                "passed": true,
                "detail": "assert first_repeated_char('aabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabac'",
                "passed": true,
                "detail": "assert first_repeated_char('aabac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabba'",
                "passed": true,
                "detail": "assert first_repeated_char('aabba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabca'",
                "passed": true,
                "detail": "assert first_repeated_char('aabca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacab'",
                "passed": true,
                "detail": "assert first_repeated_char('aacab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacac'",
                "passed": true,
                "detail": "assert first_repeated_char('aacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacba'",
                "passed": true,
                "detail": "assert first_repeated_char('aacba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacca'",
                "passed": true,
                "detail": "assert first_repeated_char('aacca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaab'",
                "passed": true,
                "detail": "assert first_repeated_char('abaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaac'",
                "passed": true,
                "detail": "assert first_repeated_char('abaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababa'",
                "passed": true,
                "detail": "assert first_repeated_char('ababa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababb'",
                "passed": true,
                "detail": "assert first_repeated_char('ababb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababc'",
                "passed": true,
                "detail": "assert first_repeated_char('ababc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaca'",
                "passed": true,
                "detail": "assert first_repeated_char('abaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacb'",
                "passed": true,
                "detail": "assert first_repeated_char('abacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacc'",
                "passed": true,
                "detail": "assert first_repeated_char('abacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbab'",
                "passed": true,
                "detail": "assert first_repeated_char('abbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbac'",
                "passed": true,
                "detail": "assert first_repeated_char('abbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbba'",
                "passed": true,
                "detail": "assert first_repeated_char('abbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbca'",
                "passed": true,
                "detail": "assert first_repeated_char('abbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcab'",
                "passed": true,
                "detail": "assert first_repeated_char('abcab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcac'",
                "passed": true,
                "detail": "assert first_repeated_char('abcac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcba'",
                "passed": true,
                "detail": "assert first_repeated_char('abcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcca'",
                "passed": true,
                "detail": "assert first_repeated_char('abcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'abccb'",
                "passed": true,
                "detail": "assert first_repeated_char('abccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'abccc'",
                "passed": true,
                "detail": "assert first_repeated_char('abccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaab'",
                "passed": true,
                "detail": "assert first_repeated_char('acaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaac'",
                "passed": true,
                "detail": "assert first_repeated_char('acaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaba'",
                "passed": true,
                "detail": "assert first_repeated_char('acaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabb'",
                "passed": true,
                "detail": "assert first_repeated_char('acabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabc'",
                "passed": true,
                "detail": "assert first_repeated_char('acabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaca'",
                "passed": true,
                "detail": "assert first_repeated_char('acaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacb'",
                "passed": true,
                "detail": "assert first_repeated_char('acacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacc'",
                "passed": true,
                "detail": "assert first_repeated_char('acacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbab'",
                "passed": true,
                "detail": "assert first_repeated_char('acbab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbac'",
                "passed": true,
                "detail": "assert first_repeated_char('acbac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbba'",
                "passed": true,
                "detail": "assert first_repeated_char('acbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbca'",
                "passed": true,
                "detail": "assert first_repeated_char('acbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accaa'",
                "passed": true,
                "detail": "assert first_repeated_char('accaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accab'",
                "passed": true,
                "detail": "assert first_repeated_char('accab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accac'",
                "passed": true,
                "detail": "assert first_repeated_char('accac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accba'",
                "passed": true,
                "detail": "assert first_repeated_char('accba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accbb'",
                "passed": true,
                "detail": "assert first_repeated_char('accbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accbc'",
                "passed": true,
                "detail": "assert first_repeated_char('accbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accca'",
                "passed": true,
                "detail": "assert first_repeated_char('accca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acccb'",
                "passed": true,
                "detail": "assert first_repeated_char('acccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acccc'",
                "passed": true,
                "detail": "assert first_repeated_char('acccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaab'",
                "passed": true,
                "detail": "assert first_repeated_char('baaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaac'",
                "passed": true,
                "detail": "assert first_repeated_char('baaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaba'",
                "passed": true,
                "detail": "assert first_repeated_char('baaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baabb'",
                "passed": true,
                "detail": "assert first_repeated_char('baabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baabc'",
                "passed": true,
                "detail": "assert first_repeated_char('baabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaca'",
                "passed": true,
                "detail": "assert first_repeated_char('baaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baacb'",
                "passed": true,
                "detail": "assert first_repeated_char('baacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baacc'",
                "passed": true,
                "detail": "assert first_repeated_char('baacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'babaa'",
                "passed": true,
                "detail": "assert first_repeated_char('babaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babab'",
                "passed": true,
                "detail": "assert first_repeated_char('babab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babac'",
                "passed": true,
                "detail": "assert first_repeated_char('babac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babba'",
                "passed": true,
                "detail": "assert first_repeated_char('babba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbb'",
                "passed": true,
                "detail": "assert first_repeated_char('babbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbc'",
                "passed": true,
                "detail": "assert first_repeated_char('babbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babca'",
                "passed": true,
                "detail": "assert first_repeated_char('babca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcb'",
                "passed": true,
                "detail": "assert first_repeated_char('babcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcc'",
                "passed": true,
                "detail": "assert first_repeated_char('babcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacab'",
                "passed": true,
                "detail": "assert first_repeated_char('bacab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacac'",
                "passed": true,
                "detail": "assert first_repeated_char('bacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacba'",
                "passed": true,
                "detail": "assert first_repeated_char('bacba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacca'",
                "passed": true,
                "detail": "assert first_repeated_char('bacca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baccb'",
                "passed": true,
                "detail": "assert first_repeated_char('baccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baccc'",
                "passed": true,
                "detail": "assert first_repeated_char('baccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccab'",
                "passed": true,
                "detail": "assert first_repeated_char('bccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccac'",
                "passed": true,
                "detail": "assert first_repeated_char('bccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccba'",
                "passed": true,
                "detail": "assert first_repeated_char('bccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccca'",
                "passed": true,
                "detail": "assert first_repeated_char('bccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaab'",
                "passed": true,
                "detail": "assert first_repeated_char('caaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaac'",
                "passed": true,
                "detail": "assert first_repeated_char('caaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaba'",
                "passed": true,
                "detail": "assert first_repeated_char('caaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabb'",
                "passed": true,
                "detail": "assert first_repeated_char('caabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabc'",
                "passed": true,
                "detail": "assert first_repeated_char('caabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaca'",
                "passed": true,
                "detail": "assert first_repeated_char('caaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caacb'",
                "passed": true,
                "detail": "assert first_repeated_char('caacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caacc'",
                "passed": true,
                "detail": "assert first_repeated_char('caacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabab'",
                "passed": true,
                "detail": "assert first_repeated_char('cabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabac'",
                "passed": true,
                "detail": "assert first_repeated_char('cabac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabba'",
                "passed": true,
                "detail": "assert first_repeated_char('cabba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabca'",
                "passed": true,
                "detail": "assert first_repeated_char('cabca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cacaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacab'",
                "passed": true,
                "detail": "assert first_repeated_char('cacab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacac'",
                "passed": true,
                "detail": "assert first_repeated_char('cacac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacba'",
                "passed": true,
                "detail": "assert first_repeated_char('cacba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacca'",
                "passed": true,
                "detail": "assert first_repeated_char('cacca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccb'",
                "passed": true,
                "detail": "assert first_repeated_char('caccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccc'",
                "passed": true,
                "detail": "assert first_repeated_char('caccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccab'",
                "passed": true,
                "detail": "assert first_repeated_char('cccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccac'",
                "passed": true,
                "detail": "assert first_repeated_char('cccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccba'",
                "passed": true,
                "detail": "assert first_repeated_char('cccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccca'",
                "passed": true,
                "detail": "assert first_repeated_char('cccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccc') == 'c'"
              },
              {
                "name": "Unicode characters",
                "passed": true,
                "detail": "assert first_repeated_char('é猫é') == 'é'"
              },
              {
                "name": "Case-sensitive identity",
                "passed": true,
                "detail": "assert first_repeated_char('aAAa') == 'A'"
              }
            ]
          }
        },
        {
          "label": "Prefix search without a set",
          "kind": "valid_alternative",
          "response": "```python\ndef first_repeated_char(s):\n    for index, c in enumerate(s):\n        if c in s[:index]:\n            return c\n    return None\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 366,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert first_repeated_char(\"abcabc\") == \"a\""
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert first_repeated_char(\"abc\") == None"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert first_repeated_char(\"123123\") == \"1\""
              },
              {
                "name": "Short-string enumeration ''",
                "passed": true,
                "detail": "assert first_repeated_char('') == None"
              },
              {
                "name": "Short-string enumeration 'a'",
                "passed": true,
                "detail": "assert first_repeated_char('a') == None"
              },
              {
                "name": "Short-string enumeration 'b'",
                "passed": true,
                "detail": "assert first_repeated_char('b') == None"
              },
              {
                "name": "Short-string enumeration 'c'",
                "passed": true,
                "detail": "assert first_repeated_char('c') == None"
              },
              {
                "name": "Short-string enumeration 'aa'",
                "passed": true,
                "detail": "assert first_repeated_char('aa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ab'",
                "passed": true,
                "detail": "assert first_repeated_char('ab') == None"
              },
              {
                "name": "Short-string enumeration 'ac'",
                "passed": true,
                "detail": "assert first_repeated_char('ac') == None"
              },
              {
                "name": "Short-string enumeration 'ba'",
                "passed": true,
                "detail": "assert first_repeated_char('ba') == None"
              },
              {
                "name": "Short-string enumeration 'bb'",
                "passed": true,
                "detail": "assert first_repeated_char('bb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bc'",
                "passed": true,
                "detail": "assert first_repeated_char('bc') == None"
              },
              {
                "name": "Short-string enumeration 'ca'",
                "passed": true,
                "detail": "assert first_repeated_char('ca') == None"
              },
              {
                "name": "Short-string enumeration 'cb'",
                "passed": true,
                "detail": "assert first_repeated_char('cb') == None"
              },
              {
                "name": "Short-string enumeration 'cc'",
                "passed": true,
                "detail": "assert first_repeated_char('cc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aab'",
                "passed": true,
                "detail": "assert first_repeated_char('aab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aac'",
                "passed": true,
                "detail": "assert first_repeated_char('aac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aba'",
                "passed": true,
                "detail": "assert first_repeated_char('aba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abb'",
                "passed": true,
                "detail": "assert first_repeated_char('abb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abc'",
                "passed": true,
                "detail": "assert first_repeated_char('abc') == None"
              },
              {
                "name": "Short-string enumeration 'aca'",
                "passed": true,
                "detail": "assert first_repeated_char('aca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acb'",
                "passed": true,
                "detail": "assert first_repeated_char('acb') == None"
              },
              {
                "name": "Short-string enumeration 'acc'",
                "passed": true,
                "detail": "assert first_repeated_char('acc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baa'",
                "passed": true,
                "detail": "assert first_repeated_char('baa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bab'",
                "passed": true,
                "detail": "assert first_repeated_char('bab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bac'",
                "passed": true,
                "detail": "assert first_repeated_char('bac') == None"
              },
              {
                "name": "Short-string enumeration 'bba'",
                "passed": true,
                "detail": "assert first_repeated_char('bba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bca'",
                "passed": true,
                "detail": "assert first_repeated_char('bca') == None"
              },
              {
                "name": "Short-string enumeration 'bcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caa'",
                "passed": true,
                "detail": "assert first_repeated_char('caa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cab'",
                "passed": true,
                "detail": "assert first_repeated_char('cab') == None"
              },
              {
                "name": "Short-string enumeration 'cac'",
                "passed": true,
                "detail": "assert first_repeated_char('cac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cba'",
                "passed": true,
                "detail": "assert first_repeated_char('cba') == None"
              },
              {
                "name": "Short-string enumeration 'cbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cca'",
                "passed": true,
                "detail": "assert first_repeated_char('cca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abab'",
                "passed": true,
                "detail": "assert first_repeated_char('abab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abac'",
                "passed": true,
                "detail": "assert first_repeated_char('abac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abba'",
                "passed": true,
                "detail": "assert first_repeated_char('abba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abca'",
                "passed": true,
                "detail": "assert first_repeated_char('abca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acab'",
                "passed": true,
                "detail": "assert first_repeated_char('acab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acac'",
                "passed": true,
                "detail": "assert first_repeated_char('acac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acba'",
                "passed": true,
                "detail": "assert first_repeated_char('acba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acca'",
                "passed": true,
                "detail": "assert first_repeated_char('acca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accb'",
                "passed": true,
                "detail": "assert first_repeated_char('accb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accc'",
                "passed": true,
                "detail": "assert first_repeated_char('accc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baab'",
                "passed": true,
                "detail": "assert first_repeated_char('baab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baac'",
                "passed": true,
                "detail": "assert first_repeated_char('baac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baba'",
                "passed": true,
                "detail": "assert first_repeated_char('baba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babb'",
                "passed": true,
                "detail": "assert first_repeated_char('babb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babc'",
                "passed": true,
                "detail": "assert first_repeated_char('babc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'baca'",
                "passed": true,
                "detail": "assert first_repeated_char('baca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caab'",
                "passed": true,
                "detail": "assert first_repeated_char('caab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caac'",
                "passed": true,
                "detail": "assert first_repeated_char('caac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caba'",
                "passed": true,
                "detail": "assert first_repeated_char('caba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caca'",
                "passed": true,
                "detail": "assert first_repeated_char('caca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabab'",
                "passed": true,
                "detail": "assert first_repeated_char('aabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabac'",
                "passed": true,
                "detail": "assert first_repeated_char('aabac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabba'",
                "passed": true,
                "detail": "assert first_repeated_char('aabba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabca'",
                "passed": true,
                "detail": "assert first_repeated_char('aabca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacab'",
                "passed": true,
                "detail": "assert first_repeated_char('aacab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacac'",
                "passed": true,
                "detail": "assert first_repeated_char('aacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacba'",
                "passed": true,
                "detail": "assert first_repeated_char('aacba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacca'",
                "passed": true,
                "detail": "assert first_repeated_char('aacca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaab'",
                "passed": true,
                "detail": "assert first_repeated_char('abaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaac'",
                "passed": true,
                "detail": "assert first_repeated_char('abaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababa'",
                "passed": true,
                "detail": "assert first_repeated_char('ababa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababb'",
                "passed": true,
                "detail": "assert first_repeated_char('ababb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababc'",
                "passed": true,
                "detail": "assert first_repeated_char('ababc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaca'",
                "passed": true,
                "detail": "assert first_repeated_char('abaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacb'",
                "passed": true,
                "detail": "assert first_repeated_char('abacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacc'",
                "passed": true,
                "detail": "assert first_repeated_char('abacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbab'",
                "passed": true,
                "detail": "assert first_repeated_char('abbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbac'",
                "passed": true,
                "detail": "assert first_repeated_char('abbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbba'",
                "passed": true,
                "detail": "assert first_repeated_char('abbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbca'",
                "passed": true,
                "detail": "assert first_repeated_char('abbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcab'",
                "passed": true,
                "detail": "assert first_repeated_char('abcab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcac'",
                "passed": true,
                "detail": "assert first_repeated_char('abcac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcba'",
                "passed": true,
                "detail": "assert first_repeated_char('abcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcca'",
                "passed": true,
                "detail": "assert first_repeated_char('abcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'abccb'",
                "passed": true,
                "detail": "assert first_repeated_char('abccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'abccc'",
                "passed": true,
                "detail": "assert first_repeated_char('abccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaab'",
                "passed": true,
                "detail": "assert first_repeated_char('acaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaac'",
                "passed": true,
                "detail": "assert first_repeated_char('acaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaba'",
                "passed": true,
                "detail": "assert first_repeated_char('acaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabb'",
                "passed": true,
                "detail": "assert first_repeated_char('acabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabc'",
                "passed": true,
                "detail": "assert first_repeated_char('acabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaca'",
                "passed": true,
                "detail": "assert first_repeated_char('acaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacb'",
                "passed": true,
                "detail": "assert first_repeated_char('acacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacc'",
                "passed": true,
                "detail": "assert first_repeated_char('acacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbab'",
                "passed": true,
                "detail": "assert first_repeated_char('acbab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbac'",
                "passed": true,
                "detail": "assert first_repeated_char('acbac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbba'",
                "passed": true,
                "detail": "assert first_repeated_char('acbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbca'",
                "passed": true,
                "detail": "assert first_repeated_char('acbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accaa'",
                "passed": true,
                "detail": "assert first_repeated_char('accaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accab'",
                "passed": true,
                "detail": "assert first_repeated_char('accab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accac'",
                "passed": true,
                "detail": "assert first_repeated_char('accac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accba'",
                "passed": true,
                "detail": "assert first_repeated_char('accba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accbb'",
                "passed": true,
                "detail": "assert first_repeated_char('accbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accbc'",
                "passed": true,
                "detail": "assert first_repeated_char('accbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accca'",
                "passed": true,
                "detail": "assert first_repeated_char('accca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acccb'",
                "passed": true,
                "detail": "assert first_repeated_char('acccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acccc'",
                "passed": true,
                "detail": "assert first_repeated_char('acccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaab'",
                "passed": true,
                "detail": "assert first_repeated_char('baaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaac'",
                "passed": true,
                "detail": "assert first_repeated_char('baaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaba'",
                "passed": true,
                "detail": "assert first_repeated_char('baaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baabb'",
                "passed": true,
                "detail": "assert first_repeated_char('baabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baabc'",
                "passed": true,
                "detail": "assert first_repeated_char('baabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaca'",
                "passed": true,
                "detail": "assert first_repeated_char('baaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baacb'",
                "passed": true,
                "detail": "assert first_repeated_char('baacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baacc'",
                "passed": true,
                "detail": "assert first_repeated_char('baacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'babaa'",
                "passed": true,
                "detail": "assert first_repeated_char('babaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babab'",
                "passed": true,
                "detail": "assert first_repeated_char('babab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babac'",
                "passed": true,
                "detail": "assert first_repeated_char('babac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babba'",
                "passed": true,
                "detail": "assert first_repeated_char('babba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbb'",
                "passed": true,
                "detail": "assert first_repeated_char('babbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbc'",
                "passed": true,
                "detail": "assert first_repeated_char('babbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babca'",
                "passed": true,
                "detail": "assert first_repeated_char('babca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcb'",
                "passed": true,
                "detail": "assert first_repeated_char('babcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcc'",
                "passed": true,
                "detail": "assert first_repeated_char('babcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacab'",
                "passed": true,
                "detail": "assert first_repeated_char('bacab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacac'",
                "passed": true,
                "detail": "assert first_repeated_char('bacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacba'",
                "passed": true,
                "detail": "assert first_repeated_char('bacba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacca'",
                "passed": true,
                "detail": "assert first_repeated_char('bacca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baccb'",
                "passed": true,
                "detail": "assert first_repeated_char('baccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baccc'",
                "passed": true,
                "detail": "assert first_repeated_char('baccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccab'",
                "passed": true,
                "detail": "assert first_repeated_char('bccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccac'",
                "passed": true,
                "detail": "assert first_repeated_char('bccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccba'",
                "passed": true,
                "detail": "assert first_repeated_char('bccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccca'",
                "passed": true,
                "detail": "assert first_repeated_char('bccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaab'",
                "passed": true,
                "detail": "assert first_repeated_char('caaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaac'",
                "passed": true,
                "detail": "assert first_repeated_char('caaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaba'",
                "passed": true,
                "detail": "assert first_repeated_char('caaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabb'",
                "passed": true,
                "detail": "assert first_repeated_char('caabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabc'",
                "passed": true,
                "detail": "assert first_repeated_char('caabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaca'",
                "passed": true,
                "detail": "assert first_repeated_char('caaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caacb'",
                "passed": true,
                "detail": "assert first_repeated_char('caacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caacc'",
                "passed": true,
                "detail": "assert first_repeated_char('caacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabab'",
                "passed": true,
                "detail": "assert first_repeated_char('cabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabac'",
                "passed": true,
                "detail": "assert first_repeated_char('cabac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabba'",
                "passed": true,
                "detail": "assert first_repeated_char('cabba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabca'",
                "passed": true,
                "detail": "assert first_repeated_char('cabca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cacaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacab'",
                "passed": true,
                "detail": "assert first_repeated_char('cacab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacac'",
                "passed": true,
                "detail": "assert first_repeated_char('cacac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacba'",
                "passed": true,
                "detail": "assert first_repeated_char('cacba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacca'",
                "passed": true,
                "detail": "assert first_repeated_char('cacca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccb'",
                "passed": true,
                "detail": "assert first_repeated_char('caccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccc'",
                "passed": true,
                "detail": "assert first_repeated_char('caccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccab'",
                "passed": true,
                "detail": "assert first_repeated_char('cccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccac'",
                "passed": true,
                "detail": "assert first_repeated_char('cccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccba'",
                "passed": true,
                "detail": "assert first_repeated_char('cccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccca'",
                "passed": true,
                "detail": "assert first_repeated_char('cccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccc') == 'c'"
              },
              {
                "name": "Unicode characters",
                "passed": true,
                "detail": "assert first_repeated_char('é猫é') == 'é'"
              },
              {
                "name": "Case-sensitive identity",
                "passed": true,
                "detail": "assert first_repeated_char('aAAa') == 'A'"
              }
            ]
          }
        },
        {
          "label": "Incorrect implementation control",
          "kind": "invalid_control",
          "response": "```python\ndef first_repeated_char(s):\n    return next((c for c in s if s.count(c) > 1), None)\n```",
          "is_full_response": true,
          "expected": "Reject: violates the behavioral contract.",
          "verification": "executed",
          "why": "The wrong implementation passes all three source tests but fails on 'abba': it returns 'a' instead of 'b'.",
          "result": {
            "passed": false,
            "failure_count": 55,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert first_repeated_char(\"abcabc\") == \"a\""
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert first_repeated_char(\"abc\") == None"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert first_repeated_char(\"123123\") == \"1\""
              },
              {
                "name": "Short-string enumeration ''",
                "passed": true,
                "detail": "assert first_repeated_char('') == None"
              },
              {
                "name": "Short-string enumeration 'a'",
                "passed": true,
                "detail": "assert first_repeated_char('a') == None"
              },
              {
                "name": "Short-string enumeration 'b'",
                "passed": true,
                "detail": "assert first_repeated_char('b') == None"
              },
              {
                "name": "Short-string enumeration 'c'",
                "passed": true,
                "detail": "assert first_repeated_char('c') == None"
              },
              {
                "name": "Short-string enumeration 'aa'",
                "passed": true,
                "detail": "assert first_repeated_char('aa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ab'",
                "passed": true,
                "detail": "assert first_repeated_char('ab') == None"
              },
              {
                "name": "Short-string enumeration 'ac'",
                "passed": true,
                "detail": "assert first_repeated_char('ac') == None"
              },
              {
                "name": "Short-string enumeration 'ba'",
                "passed": true,
                "detail": "assert first_repeated_char('ba') == None"
              },
              {
                "name": "Short-string enumeration 'bb'",
                "passed": true,
                "detail": "assert first_repeated_char('bb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bc'",
                "passed": true,
                "detail": "assert first_repeated_char('bc') == None"
              },
              {
                "name": "Short-string enumeration 'ca'",
                "passed": true,
                "detail": "assert first_repeated_char('ca') == None"
              },
              {
                "name": "Short-string enumeration 'cb'",
                "passed": true,
                "detail": "assert first_repeated_char('cb') == None"
              },
              {
                "name": "Short-string enumeration 'cc'",
                "passed": true,
                "detail": "assert first_repeated_char('cc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aab'",
                "passed": true,
                "detail": "assert first_repeated_char('aab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aac'",
                "passed": true,
                "detail": "assert first_repeated_char('aac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aba'",
                "passed": true,
                "detail": "assert first_repeated_char('aba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abb'",
                "passed": true,
                "detail": "assert first_repeated_char('abb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abc'",
                "passed": true,
                "detail": "assert first_repeated_char('abc') == None"
              },
              {
                "name": "Short-string enumeration 'aca'",
                "passed": true,
                "detail": "assert first_repeated_char('aca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acb'",
                "passed": true,
                "detail": "assert first_repeated_char('acb') == None"
              },
              {
                "name": "Short-string enumeration 'acc'",
                "passed": true,
                "detail": "assert first_repeated_char('acc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baa'",
                "passed": true,
                "detail": "assert first_repeated_char('baa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bab'",
                "passed": true,
                "detail": "assert first_repeated_char('bab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bac'",
                "passed": true,
                "detail": "assert first_repeated_char('bac') == None"
              },
              {
                "name": "Short-string enumeration 'bba'",
                "passed": true,
                "detail": "assert first_repeated_char('bba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bca'",
                "passed": true,
                "detail": "assert first_repeated_char('bca') == None"
              },
              {
                "name": "Short-string enumeration 'bcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caa'",
                "passed": true,
                "detail": "assert first_repeated_char('caa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cab'",
                "passed": true,
                "detail": "assert first_repeated_char('cab') == None"
              },
              {
                "name": "Short-string enumeration 'cac'",
                "passed": true,
                "detail": "assert first_repeated_char('cac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cba'",
                "passed": true,
                "detail": "assert first_repeated_char('cba') == None"
              },
              {
                "name": "Short-string enumeration 'cbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cca'",
                "passed": true,
                "detail": "assert first_repeated_char('cca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abab'",
                "passed": true,
                "detail": "assert first_repeated_char('abab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abac'",
                "passed": true,
                "detail": "assert first_repeated_char('abac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abba'",
                "passed": false,
                "detail": "assert first_repeated_char('abba') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abca'",
                "passed": true,
                "detail": "assert first_repeated_char('abca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acab'",
                "passed": true,
                "detail": "assert first_repeated_char('acab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acac'",
                "passed": true,
                "detail": "assert first_repeated_char('acac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acba'",
                "passed": true,
                "detail": "assert first_repeated_char('acba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acca'",
                "passed": false,
                "detail": "assert first_repeated_char('acca') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'accb'",
                "passed": true,
                "detail": "assert first_repeated_char('accb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accc'",
                "passed": true,
                "detail": "assert first_repeated_char('accc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baab'",
                "passed": false,
                "detail": "assert first_repeated_char('baab') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baac'",
                "passed": true,
                "detail": "assert first_repeated_char('baac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baba'",
                "passed": true,
                "detail": "assert first_repeated_char('baba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babb'",
                "passed": true,
                "detail": "assert first_repeated_char('babb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babc'",
                "passed": true,
                "detail": "assert first_repeated_char('babc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'baca'",
                "passed": true,
                "detail": "assert first_repeated_char('baca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccb'",
                "passed": false,
                "detail": "assert first_repeated_char('bccb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caab'",
                "passed": true,
                "detail": "assert first_repeated_char('caab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caac'",
                "passed": false,
                "detail": "assert first_repeated_char('caac') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'caba'",
                "passed": true,
                "detail": "assert first_repeated_char('caba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caca'",
                "passed": true,
                "detail": "assert first_repeated_char('caca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbc'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'aaaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaab'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaac'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaba'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaabc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaaca'",
                "passed": true,
                "detail": "assert first_repeated_char('aaaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaacc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabab'",
                "passed": true,
                "detail": "assert first_repeated_char('aabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabac'",
                "passed": true,
                "detail": "assert first_repeated_char('aabac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabba'",
                "passed": true,
                "detail": "assert first_repeated_char('aabba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabca'",
                "passed": true,
                "detail": "assert first_repeated_char('aabca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('aabcc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('aacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacab'",
                "passed": true,
                "detail": "assert first_repeated_char('aacab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacac'",
                "passed": true,
                "detail": "assert first_repeated_char('aacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacba'",
                "passed": true,
                "detail": "assert first_repeated_char('aacba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('aacbc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aacca'",
                "passed": true,
                "detail": "assert first_repeated_char('aacca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccb'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'aaccc'",
                "passed": true,
                "detail": "assert first_repeated_char('aaccc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaab'",
                "passed": true,
                "detail": "assert first_repeated_char('abaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaac'",
                "passed": true,
                "detail": "assert first_repeated_char('abaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababa'",
                "passed": true,
                "detail": "assert first_repeated_char('ababa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababb'",
                "passed": true,
                "detail": "assert first_repeated_char('ababb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'ababc'",
                "passed": true,
                "detail": "assert first_repeated_char('ababc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abaca'",
                "passed": true,
                "detail": "assert first_repeated_char('abaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacb'",
                "passed": true,
                "detail": "assert first_repeated_char('abacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abacc'",
                "passed": true,
                "detail": "assert first_repeated_char('abacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abbaa'",
                "passed": false,
                "detail": "assert first_repeated_char('abbaa') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbab'",
                "passed": false,
                "detail": "assert first_repeated_char('abbab') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbac'",
                "passed": false,
                "detail": "assert first_repeated_char('abbac') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbba'",
                "passed": false,
                "detail": "assert first_repeated_char('abbba') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbca'",
                "passed": false,
                "detail": "assert first_repeated_char('abbca') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('abbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('abcaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcab'",
                "passed": true,
                "detail": "assert first_repeated_char('abcab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcac'",
                "passed": true,
                "detail": "assert first_repeated_char('abcac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'abcba'",
                "passed": false,
                "detail": "assert first_repeated_char('abcba') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('abcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'abcca'",
                "passed": false,
                "detail": "assert first_repeated_char('abcca') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abccb'",
                "passed": false,
                "detail": "assert first_repeated_char('abccb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'abccc'",
                "passed": true,
                "detail": "assert first_repeated_char('abccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaab'",
                "passed": true,
                "detail": "assert first_repeated_char('acaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaac'",
                "passed": true,
                "detail": "assert first_repeated_char('acaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaba'",
                "passed": true,
                "detail": "assert first_repeated_char('acaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabb'",
                "passed": true,
                "detail": "assert first_repeated_char('acabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acabc'",
                "passed": true,
                "detail": "assert first_repeated_char('acabc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acaca'",
                "passed": true,
                "detail": "assert first_repeated_char('acaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacb'",
                "passed": true,
                "detail": "assert first_repeated_char('acacb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acacc'",
                "passed": true,
                "detail": "assert first_repeated_char('acacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('acbaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbab'",
                "passed": true,
                "detail": "assert first_repeated_char('acbab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbac'",
                "passed": true,
                "detail": "assert first_repeated_char('acbac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'acbba'",
                "passed": false,
                "detail": "assert first_repeated_char('acbba') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'acbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'acbbc'",
                "passed": false,
                "detail": "assert first_repeated_char('acbbc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'acbca'",
                "passed": false,
                "detail": "assert first_repeated_char('acbca') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'acbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('acbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accaa'",
                "passed": false,
                "detail": "assert first_repeated_char('accaa') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'accab'",
                "passed": false,
                "detail": "assert first_repeated_char('accab') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'accac'",
                "passed": false,
                "detail": "assert first_repeated_char('accac') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'accba'",
                "passed": false,
                "detail": "assert first_repeated_char('accba') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'accbb'",
                "passed": true,
                "detail": "assert first_repeated_char('accbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accbc'",
                "passed": true,
                "detail": "assert first_repeated_char('accbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'accca'",
                "passed": false,
                "detail": "assert first_repeated_char('accca') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'acccb'",
                "passed": true,
                "detail": "assert first_repeated_char('acccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'acccc'",
                "passed": true,
                "detail": "assert first_repeated_char('acccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'baaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('baaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaab'",
                "passed": false,
                "detail": "assert first_repeated_char('baaab') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baaac'",
                "passed": true,
                "detail": "assert first_repeated_char('baaac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baaba'",
                "passed": false,
                "detail": "assert first_repeated_char('baaba') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baabb'",
                "passed": false,
                "detail": "assert first_repeated_char('baabb') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baabc'",
                "passed": false,
                "detail": "assert first_repeated_char('baabc') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baaca'",
                "passed": true,
                "detail": "assert first_repeated_char('baaca') == 'a'"
              },
              {
                "name": "Short-string enumeration 'baacb'",
                "passed": false,
                "detail": "assert first_repeated_char('baacb') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baacc'",
                "passed": true,
                "detail": "assert first_repeated_char('baacc') == 'a'"
              },
              {
                "name": "Short-string enumeration 'babaa'",
                "passed": true,
                "detail": "assert first_repeated_char('babaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babab'",
                "passed": true,
                "detail": "assert first_repeated_char('babab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babac'",
                "passed": true,
                "detail": "assert first_repeated_char('babac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babba'",
                "passed": true,
                "detail": "assert first_repeated_char('babba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbb'",
                "passed": true,
                "detail": "assert first_repeated_char('babbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babbc'",
                "passed": true,
                "detail": "assert first_repeated_char('babbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babca'",
                "passed": true,
                "detail": "assert first_repeated_char('babca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcb'",
                "passed": true,
                "detail": "assert first_repeated_char('babcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'babcc'",
                "passed": true,
                "detail": "assert first_repeated_char('babcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bacaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacab'",
                "passed": false,
                "detail": "assert first_repeated_char('bacab') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bacac'",
                "passed": true,
                "detail": "assert first_repeated_char('bacac') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bacba'",
                "passed": true,
                "detail": "assert first_repeated_char('bacba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bacbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bacca'",
                "passed": false,
                "detail": "assert first_repeated_char('bacca') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baccb'",
                "passed": false,
                "detail": "assert first_repeated_char('baccb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'baccc'",
                "passed": true,
                "detail": "assert first_repeated_char('baccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbaca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbacc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('bbcca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bbccc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'bcaab'",
                "passed": false,
                "detail": "assert first_repeated_char('bcaab') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bcaac'",
                "passed": false,
                "detail": "assert first_repeated_char('bcaac') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bcaba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcabc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcabc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcaca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcacb'",
                "passed": false,
                "detail": "assert first_repeated_char('bcacb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bcacc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbab'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbac'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbac') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbba'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbbc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbca'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbca') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bcbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcbcc') == 'b'"
              },
              {
                "name": "Short-string enumeration 'bccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('bccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccab'",
                "passed": false,
                "detail": "assert first_repeated_char('bccab') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bccac'",
                "passed": true,
                "detail": "assert first_repeated_char('bccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bccba'",
                "passed": false,
                "detail": "assert first_repeated_char('bccba') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bccbb'",
                "passed": false,
                "detail": "assert first_repeated_char('bccbb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bccbc'",
                "passed": false,
                "detail": "assert first_repeated_char('bccbc') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bccca'",
                "passed": true,
                "detail": "assert first_repeated_char('bccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'bcccb'",
                "passed": false,
                "detail": "assert first_repeated_char('bcccb') == 'c' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'bcccc'",
                "passed": true,
                "detail": "assert first_repeated_char('bcccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('caaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaab'",
                "passed": true,
                "detail": "assert first_repeated_char('caaab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caaac'",
                "passed": false,
                "detail": "assert first_repeated_char('caaac') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'caaba'",
                "passed": true,
                "detail": "assert first_repeated_char('caaba') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabb'",
                "passed": true,
                "detail": "assert first_repeated_char('caabb') == 'a'"
              },
              {
                "name": "Short-string enumeration 'caabc'",
                "passed": false,
                "detail": "assert first_repeated_char('caabc') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'caaca'",
                "passed": false,
                "detail": "assert first_repeated_char('caaca') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'caacb'",
                "passed": false,
                "detail": "assert first_repeated_char('caacb') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'caacc'",
                "passed": false,
                "detail": "assert first_repeated_char('caacc') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cabaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cabaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabab'",
                "passed": true,
                "detail": "assert first_repeated_char('cabab') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cabac'",
                "passed": false,
                "detail": "assert first_repeated_char('cabac') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cabba'",
                "passed": false,
                "detail": "assert first_repeated_char('cabba') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cabbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cabbc'",
                "passed": false,
                "detail": "assert first_repeated_char('cabbc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cabca'",
                "passed": true,
                "detail": "assert first_repeated_char('cabca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcb'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cabcc'",
                "passed": true,
                "detail": "assert first_repeated_char('cabcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cacaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacab'",
                "passed": true,
                "detail": "assert first_repeated_char('cacab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacac'",
                "passed": true,
                "detail": "assert first_repeated_char('cacac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacba'",
                "passed": true,
                "detail": "assert first_repeated_char('cacba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cacbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cacca'",
                "passed": true,
                "detail": "assert first_repeated_char('cacca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccb'",
                "passed": true,
                "detail": "assert first_repeated_char('caccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'caccc'",
                "passed": true,
                "detail": "assert first_repeated_char('caccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaaa') == 'a'"
              },
              {
                "name": "Short-string enumeration 'cbaab'",
                "passed": false,
                "detail": "assert first_repeated_char('cbaab') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbaac'",
                "passed": false,
                "detail": "assert first_repeated_char('cbaac') == 'a' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbaba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbabb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbabc'",
                "passed": false,
                "detail": "assert first_repeated_char('cbabc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbaca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbacc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbaa') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbab') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbac'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbac') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbbba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbba') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbbbb') == 'b'"
              },
              {
                "name": "Short-string enumeration 'cbbbc'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbbc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbbca'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbca') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbbcb'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbcb') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbbcc'",
                "passed": false,
                "detail": "assert first_repeated_char('cbbcc') == 'b' -> AssertionError"
              },
              {
                "name": "Short-string enumeration 'cbcaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcab'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcac'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcba'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbcca'",
                "passed": true,
                "detail": "assert first_repeated_char('cbcca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccb'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cbccc'",
                "passed": true,
                "detail": "assert first_repeated_char('cbccc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccabc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccabc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccaca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccaca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccacc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccacc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbaa'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbab'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbac'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbba'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbbc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbca'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccbcc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccbcc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccaa'",
                "passed": true,
                "detail": "assert first_repeated_char('cccaa') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccab'",
                "passed": true,
                "detail": "assert first_repeated_char('cccab') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccac'",
                "passed": true,
                "detail": "assert first_repeated_char('cccac') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccba'",
                "passed": true,
                "detail": "assert first_repeated_char('cccba') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbb'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccbc'",
                "passed": true,
                "detail": "assert first_repeated_char('cccbc') == 'c'"
              },
              {
                "name": "Short-string enumeration 'cccca'",
                "passed": true,
                "detail": "assert first_repeated_char('cccca') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccb'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccb') == 'c'"
              },
              {
                "name": "Short-string enumeration 'ccccc'",
                "passed": true,
                "detail": "assert first_repeated_char('ccccc') == 'c'"
              },
              {
                "name": "Unicode characters",
                "passed": true,
                "detail": "assert first_repeated_char('é猫é') == 'é'"
              },
              {
                "name": "Case-sensitive identity",
                "passed": false,
                "detail": "assert first_repeated_char('aAAa') == 'A' -> AssertionError"
              }
            ]
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Python subprocess execution against source assertions and enumerated supplemental contract cases.",
        "checks": [
          {
            "name": "Seen-set scan",
            "passed": true,
            "detail": "Passed 3 source tests and 366 supplemental cases."
          },
          {
            "name": "Prefix search without a set",
            "passed": true,
            "detail": "Passed 3 source tests and 366 supplemental cases."
          },
          {
            "name": "Incorrect control is rejected",
            "passed": true,
            "detail": "Control fails 55 of 369 cases. First failure: assert first_repeated_char('abba') == 'b' -> AssertionError"
          },
          {
            "name": "Source reference audit",
            "passed": true,
            "detail": "369/369 cases pass. The wrong implementation passes all three source tests but fails on 'abba': it returns 'a' instead of 'b'."
          }
        ],
        "limitations": [
          "No separate model rubric judge was run.",
          "No performance requirement is imposed because the task does not request one.",
          "Ground-truth tests and source code can contain defects; the prime example deliberately retains a failed source reference."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "verified:mbpp:repair:635",
      "title": "Repair a sorting function that loses duplicates",
      "source": {
        "dataset": "google-research-datasets/mbpp",
        "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
        "license": "CC-BY-4.0",
        "url": "https://huggingface.co/datasets/google-research-datasets/mbpp/blob/4bb6404fdc6cacfda99d4ac4205087b89d32030c/sanitized/train-00000-of-00001.parquet",
        "split": "train",
        "row_id": "635",
        "pool": "proposed",
        "file_sha256": "d95f8ad6d2fff08fe4826122d6e3e31f75716825d0c5c340d297aca5e9e0de0e",
        "row_sha256": "4b59ec0cccea902e41126df0f958bcf50685575f7add1974604ad592ed4e0ff2",
        "selection": "Purposive demonstration; not representative or admitted for training.",
        "attribution": "Jacob Austin et al., Program Synthesis with Large Language Models (2021), Google Research MBPP. Adapted into repair prompts; rubrics, alternative solutions and supplemental tests added for this gallery."
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "software_repair",
        "leaf": "software_repair.functional_bug_repair",
        "confidence": "high",
        "reason": "Explicitly adapted from a function-writing task into a repair task; original prompt retained below."
      },
      "language": "en",
      "prompt": "Repair this Python implementation. Return a list containing all input numbers in nondecreasing order, preserving multiplicity. Empty input returns an empty list. Keep the heap_sort(iterable) call signature; despite that name, no particular sorting algorithm is required.\n\n```python\ndef heap_sort(iterable):\n    return sorted(set(iterable))\n```",
      "original_prompt": "Write a function to sort the given list.",
      "context": [
        "Original MBPP prompt: Write a function to sort the given list.",
        "Task adaptation: a faulty implementation and explicit callable/behavior contract were added for this gallery. Grounding distinguishes source tests from supplemental checks."
      ],
      "environment_note": "Self-contained Python. The displayed implementations were executed locally; no external repository is required.",
      "rubric": {
        "markdown": "# Repair a sorting function that loses duplicates\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Correct behavior on source cases\n\n**ID:** source\\_behavior\n\n**Criterion:** Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** All source examples pass with the requested callable.\n- **0:** One or more source examples fail or the callable cannot be invoked.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"source\\_tests\", \"adapted\\_task\\_contract\"\\]\n\n### Generalizes across the stated contract\n\n**ID:** general\\_behavior\n\n**Criterion:** Produces a nondecreasing list containing exactly the input multiset. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** All supplemental contract checks pass.\n- **0:** A supplemental case violates the contract.\n\n**Tags:** \\[\"Correctness\", \"Solution diversity\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"supplemental\\_tests\", \"adapted\\_task\\_contract\"\\]\n",
        "raw": {
          "title": "Repair a sorting function that loses duplicates",
          "scoring": "positive_weights",
          "criteria": [
            {
              "id": "source_behavior",
              "criterion": "Implements the required callable and returns correct results on the source examples. Score observable behavior; no source-code or algorithm similarity is required.",
              "weight": 3.0,
              "grading_anchors": {
                "1": "All source examples pass with the requested callable.",
                "0": "One or more source examples fail or the callable cannot be invoked."
              },
              "evidence_refs": [
                "source_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            },
            {
              "id": "general_behavior",
              "criterion": "Produces a nondecreasing list containing exactly the input multiset. The supplemental cases operationalize this contract; memorizing the source examples is insufficient.",
              "weight": 5.0,
              "grading_anchors": {
                "1": "All supplemental contract checks pass.",
                "0": "A supplemental case violates the contract."
              },
              "evidence_refs": [
                "supplemental_tests",
                "adapted_task_contract"
              ],
              "status": "provisional",
              "method": "rule",
              "gate": false
            }
          ]
        },
        "origin": "drafted",
        "grounding": "reference_solution_and_executable_checks",
        "criterion_count": 2,
        "presentation": {
          "source_behavior": {
            "title": "Correct behavior on source cases",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          },
          "general_behavior": {
            "title": "Generalizes across the stated contract",
            "tags": [
              "Correctness",
              "Solution diversity"
            ]
          }
        },
        "conversion_note": "Editorial draft from verified outcomes; not a source-provided or independently judged rubric. Acceptance in this demonstration requires all behavioral criteria to pass; weighted partial credit is not an acceptance decision."
      },
      "reference": {
        "answer": "Function behavior defined by the explicit contract and tests.",
        "solution": "import heapq as hq\ndef heap_sort(iterable):\n    h = []\n    for value in iterable:\n        hq.heappush(h, value)\n    return [hq.heappop(h) for i in range(len(h))]",
        "tests": [
          "assert heap_sort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
          "assert heap_sort([25, 35, 22, 85, 14, 65, 75, 25, 58])==[14, 22, 25, 25, 35, 58, 65, 75, 85]",
          "assert heap_sort( [7, 1, 9, 5])==[1,5,7,9]"
        ],
        "source_reference_verified": true,
        "supplemental_test_count": 364,
        "warning": "A reference using heapq does not justify a mandatory heapq criterion. Both built-in sorting and insertion sort satisfy the behavior."
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Two different valid implementations pass the same checks.",
          "Incorrect controls expose failure modes and sparse-test blind spots.",
          "A reference using heapq does not justify a mandatory heapq criterion. Both built-in sorting and insertion sort satisfy the behavior."
        ],
        "risks": [
          "Finite tests do not prove correctness for every input.",
          "The explicit repair contract and supplemental tests are authored additions, not original source annotations."
        ],
        "must_hold": [
          "Output is sorted and contains every input value with the original multiplicity.",
          "Empty, repeated, negative, and already sorted inputs satisfy the same contract."
        ],
        "may_vary": [
          "Built-in sort, heap sort, insertion sort, or another correct sorting algorithm.",
          "Variable names, formatting, comments, and data structures are optional choices."
        ],
        "proposed_changes": [
          "Keep source reference code as one audited implementation, never as the required method.",
          "Expand behavior checks before scaling admission; use property-based and adversarial tests."
        ]
      },
      "probes": [
        {
          "label": "Python's general-purpose sort",
          "kind": "valid_alternative",
          "response": "```python\ndef heap_sort(iterable):\n    return sorted(iterable)\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 364,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert heap_sort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert heap_sort([25, 35, 22, 85, 14, 65, 75, 25, 58])==[14, 22, 25, 25, 35, 58, 65, 75, 85]"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert heap_sort( [7, 1, 9, 5])==[1,5,7,9]"
              },
              {
                "name": "Enumerated input []",
                "passed": true,
                "detail": "assert heap_sort([]) == []"
              },
              {
                "name": "Enumerated input [-1]",
                "passed": true,
                "detail": "assert heap_sort([-1]) == [-1]"
              },
              {
                "name": "Enumerated input [0]",
                "passed": true,
                "detail": "assert heap_sort([0]) == [0]"
              },
              {
                "name": "Enumerated input [1]",
                "passed": true,
                "detail": "assert heap_sort([1]) == [1]"
              },
              {
                "name": "Enumerated input [-1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1]) == [-1, -1]"
              },
              {
                "name": "Enumerated input [-1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [-1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0]) == [0, 0]"
              },
              {
                "name": "Enumerated input [0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1]) == [1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1]) == [-1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0]) == [0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1]) == [1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1]) == [-1, -1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0]) == [0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1]) == [1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, -1]) == [-1, -1, -1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, 0]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, 1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, -1]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, 0]) == [0, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, 1]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, -1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, 0]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, 1]) == [1, 1, 1, 1, 1]"
              }
            ]
          }
        },
        {
          "label": "Insertion sort",
          "kind": "valid_alternative",
          "response": "```python\ndef heap_sort(iterable):\n    out = []\n    for value in iterable:\n        i = len(out)\n        out.append(value)\n        while i > 0 and out[i - 1] > value:\n            out[i] = out[i - 1]\n            i -= 1\n        out[i] = value\n    return out\n```",
          "is_full_response": true,
          "expected": "Accept: all tested behaviors hold; algorithm and style are allowed to differ.",
          "verification": "executed",
          "why": "Executed against source and supplemental tests; this is finite evidence, not a proof over all inputs.",
          "result": {
            "passed": true,
            "source_tests": 3,
            "supplemental_tests": 364,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert heap_sort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
              },
              {
                "name": "Source test 2",
                "passed": true,
                "detail": "assert heap_sort([25, 35, 22, 85, 14, 65, 75, 25, 58])==[14, 22, 25, 25, 35, 58, 65, 75, 85]"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert heap_sort( [7, 1, 9, 5])==[1,5,7,9]"
              },
              {
                "name": "Enumerated input []",
                "passed": true,
                "detail": "assert heap_sort([]) == []"
              },
              {
                "name": "Enumerated input [-1]",
                "passed": true,
                "detail": "assert heap_sort([-1]) == [-1]"
              },
              {
                "name": "Enumerated input [0]",
                "passed": true,
                "detail": "assert heap_sort([0]) == [0]"
              },
              {
                "name": "Enumerated input [1]",
                "passed": true,
                "detail": "assert heap_sort([1]) == [1]"
              },
              {
                "name": "Enumerated input [-1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1]) == [-1, -1]"
              },
              {
                "name": "Enumerated input [-1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [-1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0]) == [0, 0]"
              },
              {
                "name": "Enumerated input [0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1]) == [1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1]) == [-1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1]) == [-1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1]) == [-1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0]) == [0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1]) == [-1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0]) == [0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1]) == [-1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0]) == [0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1]) == [1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1]) == [-1, -1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1]) == [-1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1]) == [-1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1]) == [-1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0]) == [0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1]) == [-1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1]) == [-1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1]) == [-1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0]) == [0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1]) == [-1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1]) == [-1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0]) == [0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1]) == [-1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0]) == [0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1]) == [1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, -1]) == [-1, -1, -1, -1, -1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, 0]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, -1, 1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 0, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, -1, 1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 0, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, -1, 1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, -1, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 0, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, -1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, -1]) == [-1, -1, -1, -1, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, 0]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, -1, 1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 0, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, -1, 1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 0, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, -1]) == [-1, -1, -1, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, 0]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, -1, 1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 0, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, -1, 1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, -1]) == [-1, -1, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, 0]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, -1, 1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, -1]) == [-1, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, 0]) == [0, 0, 0, 0, 0]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 0, 1]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 0, 1, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 0, 1, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 0, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, 1, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, -1]) == [-1, -1, -1, -1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, 0]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, -1, 1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 0, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, -1, 1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, -1]) == [-1, -1, -1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, 0]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, -1, 1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 0, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1, 1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, -1]) == [-1, -1, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, 0]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, -1, 1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, -1]) == [-1, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, 0]) == [0, 0, 0, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 0, 1]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 0, 1, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, 1, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, -1]) == [-1, -1, -1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, 0]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, -1, 1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 0, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, -1, 1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, -1]) == [-1, -1, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, 0]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, -1, 1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, -1]) == [-1, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, 0]) == [0, 0, 0, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 0, 1]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 0, 1, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, -1]) == [-1, -1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, 0]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, -1, 1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, -1]) == [-1, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, 0]) == [0, 0, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 0, 1]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, -1]) == [-1, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, 0]) == [0, 1, 1, 1, 1]"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 1]",
                "passed": true,
                "detail": "assert heap_sort([1, 1, 1, 1, 1]) == [1, 1, 1, 1, 1]"
              }
            ]
          }
        },
        {
          "label": "Incorrect implementation control",
          "kind": "invalid_control",
          "response": "```python\ndef heap_sort(iterable):\n    return sorted(set(iterable))\n```",
          "is_full_response": true,
          "expected": "Reject: violates the behavioral contract.",
          "verification": "executed",
          "why": "A reference using heapq does not justify a mandatory heapq criterion. Both built-in sorting and insertion sort satisfy the behavior.",
          "result": {
            "passed": false,
            "failure_count": 349,
            "checks": [
              {
                "name": "Source test 1",
                "passed": true,
                "detail": "assert heap_sort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
              },
              {
                "name": "Source test 2",
                "passed": false,
                "detail": "assert heap_sort([25, 35, 22, 85, 14, 65, 75, 25, 58])==[14, 22, 25, 25, 35, 58, 65, 75, 85] -> AssertionError"
              },
              {
                "name": "Source test 3",
                "passed": true,
                "detail": "assert heap_sort( [7, 1, 9, 5])==[1,5,7,9]"
              },
              {
                "name": "Enumerated input []",
                "passed": true,
                "detail": "assert heap_sort([]) == []"
              },
              {
                "name": "Enumerated input [-1]",
                "passed": true,
                "detail": "assert heap_sort([-1]) == [-1]"
              },
              {
                "name": "Enumerated input [0]",
                "passed": true,
                "detail": "assert heap_sort([0]) == [0]"
              },
              {
                "name": "Enumerated input [1]",
                "passed": true,
                "detail": "assert heap_sort([1]) == [1]"
              },
              {
                "name": "Enumerated input [-1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1]) == [-1, -1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [-1, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [0, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1]) == [-1, 0]"
              },
              {
                "name": "Enumerated input [0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0]) == [0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, -1]) == [-1, 1]"
              },
              {
                "name": "Enumerated input [1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, 0]) == [0, 1]"
              },
              {
                "name": "Enumerated input [1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1]) == [1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1]) == [-1, -1, -1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0]) == [-1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1]) == [-1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1]) == [-1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0]) == [-1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1]",
                "passed": true,
                "detail": "assert heap_sort([-1, 0, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1]) == [-1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0]",
                "passed": true,
                "detail": "assert heap_sort([-1, 1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [-1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1]) == [-1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1]) == [-1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0]) == [-1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1]",
                "passed": true,
                "detail": "assert heap_sort([0, -1, 1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1]) == [-1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0]) == [0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1]) == [0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1]",
                "passed": true,
                "detail": "assert heap_sort([0, 1, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0]) == [0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1]) == [0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1]) == [-1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0]",
                "passed": true,
                "detail": "assert heap_sort([1, -1, 0]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1]) == [-1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1]",
                "passed": true,
                "detail": "assert heap_sort([1, 0, -1]) == [-1, 0, 1]"
              },
              {
                "name": "Enumerated input [1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0]) == [0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1]) == [0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1]) == [-1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0]) == [0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1]) == [1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, -1]) == [-1, -1, -1, -1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 0]) == [-1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 1]) == [-1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, -1]) == [-1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 0]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, -1]) == [-1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 0]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, -1]) == [-1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 0]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, -1]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 0]) == [-1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, -1]) == [-1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 0]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, -1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 0]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 1]) == [-1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, -1]) == [-1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 0]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, -1]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 0]) == [-1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, -1]) == [-1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 0]) == [-1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, -1]) == [-1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 0]) == [0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 1]) == [0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, -1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 0]) == [0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 1]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, -1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 0]) == [0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 1]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, -1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 0]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 1]) == [0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, -1]) == [-1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 0]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, -1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 0]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 1]) == [-1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, -1]) == [-1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 0]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, -1]) == [-1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 0]) == [0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 1]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, -1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 0]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 1]) == [0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, -1]) == [-1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 0]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 1]) == [-1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, -1]) == [-1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 0]) == [0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 1]) == [0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, -1]) == [-1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 0]) == [0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 1]) == [1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, -1, -1]) == [-1, -1, -1, -1, -1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, -1, 0]) == [-1, -1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, -1, 1]) == [-1, -1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 0, -1]) == [-1, -1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 0, 0]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 0, 1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 1, -1]) == [-1, -1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 1, 0]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, -1, 1, 1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, -1, -1]) == [-1, -1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, -1, 0]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, -1, 1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 0, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 0, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 0, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 0, 1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, -1, -1]) == [-1, -1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, -1, 0]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, -1, 1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 0, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 0, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 0, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, -1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, -1, 1, 1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, -1, -1]) == [-1, -1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, -1, 0]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, -1, 1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 0, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 0, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 0, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, -1, 1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, -1, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, -1, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, -1, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 0, -1]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 0, 0]) == [-1, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 0, 1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 0, 1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 0, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 0, 1, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, -1, -1]) == [-1, -1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, -1, 0]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, -1, 1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 0, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 0, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 0, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, -1, 1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 0, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, -1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, -1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, -1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 0, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 0, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 0, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 1, -1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 1, 0]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [-1, 1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([-1, 1, 1, 1, 1]) == [-1, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, -1, -1]) == [-1, -1, -1, -1, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, -1, 0]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, -1, 1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 0, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 0, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 0, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, -1, 1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, -1, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, -1, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, -1, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 0, -1]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 0, 0]) == [-1, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 0, 1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 0, 1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, -1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, -1, 1, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, -1, -1]) == [-1, -1, -1, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, -1, 0]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, -1, 1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 0, -1]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 0, 0]) == [-1, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 0, 1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, -1, 1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, -1, -1]) == [-1, -1, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, -1, 0]) == [-1, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, -1, 1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 0, -1]) == [-1, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 0, 0]) == [0, 0, 0, 0, 0] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 0, 1]) == [0, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 1, -1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 1, 0]) == [0, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 0, 1, 1]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, -1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, -1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, -1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 0, -1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 0, 0]) == [0, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 0, 1]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 1, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 1, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 0, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 0, 1, 1, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, -1, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, -1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, -1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, -1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 0, -1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 0, 0]) == [0, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 0, 1]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 1, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 1, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 0, 1, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, -1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, -1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, -1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 0, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 0, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 0, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 1, -1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 1, 0]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [0, 1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([0, 1, 1, 1, 1]) == [0, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, -1, -1]) == [-1, -1, -1, -1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, -1, 0]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, -1, 1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 0, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 0, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 0, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, -1, 1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 0, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, -1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, -1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, -1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 0, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 0, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 0, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 1, -1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 1, 0]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, -1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, -1, 1, 1, 1]) == [-1, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, -1, -1]) == [-1, -1, -1, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, -1, 0]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, -1, 1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 0, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 0, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 0, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, -1, 1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, -1, -1]) == [-1, -1, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, -1, 0]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, -1, 1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 0, -1]) == [-1, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 0, 0]) == [0, 0, 0, 0, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 0, 1]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 1, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 1, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 0, 1, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, -1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, -1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, -1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 0, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 0, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 0, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 1, -1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 1, 0]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 0, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 0, 1, 1, 1]) == [0, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, -1, -1]) == [-1, -1, -1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, -1, 0]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, -1, 1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 0, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 0, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 0, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 1, -1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 1, 0]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, -1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, -1, 1, 1]) == [-1, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, -1, -1]) == [-1, -1, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, -1, 0]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, -1, 1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 0, -1]) == [-1, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 0, 0]) == [0, 0, 0, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 0, 1]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 1, -1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 1, 0]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 0, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 0, 1, 1]) == [0, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, -1, -1]) == [-1, -1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, -1, 0]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, -1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, -1, 1]) == [-1, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 0, -1]) == [-1, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 0, 0]) == [0, 0, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 0, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 0, 1]) == [0, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, -1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 1, -1]) == [-1, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 0]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 1, 0]) == [0, 1, 1, 1, 1] -> AssertionError"
              },
              {
                "name": "Enumerated input [1, 1, 1, 1, 1]",
                "passed": false,
                "detail": "assert heap_sort([1, 1, 1, 1, 1]) == [1, 1, 1, 1, 1] -> AssertionError"
              }
            ]
          }
        }
      ],
      "verification": {
        "status": "executed",
        "method": "Python subprocess execution against source assertions and enumerated supplemental contract cases.",
        "checks": [
          {
            "name": "Python's general-purpose sort",
            "passed": true,
            "detail": "Passed 3 source tests and 364 supplemental cases."
          },
          {
            "name": "Insertion sort",
            "passed": true,
            "detail": "Passed 3 source tests and 364 supplemental cases."
          },
          {
            "name": "Incorrect control is rejected",
            "passed": true,
            "detail": "Control fails 349 of 367 cases. First failure: assert heap_sort([25, 35, 22, 85, 14, 65, 75, 25, 58])==[14, 22, 25, 25, 35, 58, 65, 75, 85] -> AssertionError"
          },
          {
            "name": "Source reference audit",
            "passed": true,
            "detail": "367/367 cases pass. A reference using heapq does not justify a mandatory heapq criterion. Both built-in sorting and insertion sort satisfy the behavior."
          }
        ],
        "limitations": [
          "No separate model rubric judge was run.",
          "No performance requirement is imposed because the task does not request one.",
          "Ground-truth tests and source code can contain defects; the prime example deliberately retains a failed source reference."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:mimo:s3k_1447_healthcare_ops_zh_t2_rl_006",
      "title": "会话终止：操作结果与权限范围守恒",
      "source": {
        "dataset": "XiaomiMiMo/MiMo-V2.6-RL-oss",
        "source_name": "mimo",
        "url": "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss",
        "revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "original_inventory_revision": "639865fd3374018d6cb29b9fb82dd531406fcf5f",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Original inventory verified Hugging Face commit",
        "split": "general/train",
        "row_id": "s3k_1447_healthcare_ops_zh_t2_rl_006",
        "license": "Apache-2.0",
        "pool": "current",
        "source_content_sha256": "ade5348ace27b5b7891b2b70beaf8174a6d133d1bc82b7b4b1d355d7a530e616",
        "source_record_fingerprint": "1c094e552d278b393bc154b3c8ce4b289b6e5d037ab9dded45a84fc23d3fea4b",
        "source_row_index": 902,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": false,
        "synthetic_claim_from_import": true,
        "attribution": "Source task and rubric: mimo; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "terminal_data_pipelines",
        "leaf": "terminal_data_pipelines.runtime_diagnosis_recovery",
        "confidence": "editorial_provisional",
        "reason": "Success requires terminating active sessions while preserving revoked authorization and unrelated state."
      },
      "language": "zh",
      "prompt": "user: OSU-CHS 外网 Web 服务器访问控制整改还留有一项需要收尾的会话清理工作。请先结合 workspace 中的《OSU-CHS_ePHI权限配置整改追踪表.xlsx》与《OSU-CHS_Web服务器访问日志核查明细.csv》理解涉事系统、组织归属和人员/厂商授权的业务口径，再到 SailPoint IdentityIQ 中核对当前授权状态。\n\n请找出 OSU-CHS 外网 Web 服务器上已经不再获准、但仍保持活跃连接的两条授权，确认它们分别属于人员授权和厂商远程维护授权。随后只终止这两条授权的活跃会话：不得重新授予权限，不得扩大或缩小访问范围，不得改变原有的撤权状态，也不要影响其他机构、其他资产或其他授权。\n\n完成后，请在最终回复中说明：你从资料和系统核对出的两条授权及其人员/厂商属性；操作前后两条记录的关键状态（授权是否仍被撤回、会话是否仍活跃）；以及你实际执行的会话终止操作和未修改其他授权的结论。",
      "context": [],
      "environment_note": "Original remote workspace and tools are required and are not included or executed. Response probes compare representations against source anchors only; no environment success is claimed.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Source executable check: check\\_target\\_sessions (opaque; requires the original verifier).\n\n**ID:** target\\_sessions\\_terminated\n\n**Criterion:** Source executable check: check\\_target\\_sessions (opaque; requires the original verifier).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Terminal and data pipelines\", \"zh\", \"Source criterion\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否识别出两条目标授权分别为人员授权与厂商远程维护授权，并准确说明两条记录操作前为已撤权但会话活跃、操作后仍为已撤权且会话已终止？\n\n**ID:** target\\_identity\\_and\\_before\\_after\n\n**Criterion:** 最终回复是否识别出两条目标授权分别为人员授权与厂商远程维护授权，并准确说明两条记录操作前为已撤权但会话活跃、操作后仍为已撤权且会话已终止？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实数据锚点：目标授权集合为 ENT-2016-00417（principal\\_type=workforce，principal\\_id=osu\\\\j.mitchell，access\\_scope=apache configuration and production log read）与 ENT-2016-00503（principal\\_type=vendor，principal\\_id=osu\\\\r.sanders，access\\_scope=remote web maintenance shell）。两条操作前 authorization\\_status 均为 revoked、active\\_session\\_flag 均为 1；正确后态为 authorization\\_status 仍为 revoked、active\\_session\\_flag 均为 0。回答可用中文业务表述或给出这些标识，不要求固定格式。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Terminal and data pipelines\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### 最终回复是否明确说明实际执行的是仅终止这两条已撤权授权的活跃会话，并确认未改变授权状态、访问范围或其他机构/其他授权？\n\n**ID:** operation\\_scope\\_reported\n\n**Criterion:** 最终回复是否明确说明实际执行的是仅终止这两条已撤权授权的活跃会话，并确认未改变授权状态、访问范围或其他机构/其他授权？\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** 真实操作口径：仅对 ENT-2016-00417 与 ENT-2016-00503 将活跃会话清零；两条的 authorization\\_status 继续为 revoked，access\\_scope、其他授权及非目标机构不应被修改。最终回复需报告已执行的会话终止及范围守恒结论，不能只给出计划。\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Terminal and data pipelines\", \"zh\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Source executable check: check\\_non\\_target\\_preserved (opaque; requires the original verifier).\n\n**ID:** non\\_target\\_preserved\n\n**Criterion:** Source executable check: check\\_non\\_target\\_preserved (opaque; requires the original verifier).\n\n**Points:** +1\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"Terminal and data pipelines\", \"zh\", \"Source criterion\"\\]\n\n**Method:** rule\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "Source executable check: check_target_sessions (opaque; requires the original verifier).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "target_sessions_terminated",
              "method": "rule",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否识别出两条目标授权分别为人员授权与厂商远程维护授权，并准确说明两条记录操作前为已撤权但会话活跃、操作后仍为已撤权且会话已终止？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实数据锚点：目标授权集合为 ENT-2016-00417（principal_type=workforce，principal_id=osu\\j.mitchell，access_scope=apache configuration and production log read）与 ENT-2016-00503（principal_type=vendor，principal_id=osu\\r.sanders，access_scope=remote web maintenance shell）。两条操作前 authorization_status 均为 revoked、active_session_flag 均为 1；正确后态为 authorization_status 仍为 revoked、active_session_flag 均为 0。回答可用中文业务表述或给出这些标识，不要求固定格式。"
              },
              "id": "target_identity_and_before_after",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "最终回复是否明确说明实际执行的是仅终止这两条已撤权授权的活跃会话，并确认未改变授权状态、访问范围或其他机构/其他授权？",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "真实操作口径：仅对 ENT-2016-00417 与 ENT-2016-00503 将活跃会话清零；两条的 authorization_status 继续为 revoked，access_scope、其他授权及非目标机构不应被修改。最终回复需报告已执行的会话终止及范围守恒结论，不能只给出计划。"
              },
              "id": "operation_scope_reported",
              "method": "llm",
              "status": "provisional",
              "weight": 1.0
            },
            {
              "criterion": "Source executable check: check_non_target_preserved (opaque; requires the original verifier).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "non_target_preserved",
              "method": "rule",
              "status": "provisional",
              "weight": 1.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 4,
        "source_original_items": [
          {
            "fn": "check_target_sessions",
            "ground_truth": {
              "active_session_flag": 0,
              "authorization_status": "revoked",
              "entitlement_ids": [
                "ENT-2016-00417",
                "ENT-2016-00503"
              ]
            },
            "id": "target_sessions_terminated",
            "method": "rule",
            "question": "",
            "tier": "critical"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "target_identity_and_before_after",
            "method": "llm",
            "pass_anchor": "真实数据锚点：目标授权集合为 ENT-2016-00417（principal_type=workforce，principal_id=osu\\j.mitchell，access_scope=apache configuration and production log read）与 ENT-2016-00503（principal_type=vendor，principal_id=osu\\r.sanders，access_scope=remote web maintenance shell）。两条操作前 authorization_status 均为 revoked、active_session_flag 均为 1；正确后态为 authorization_status 仍为 revoked、active_session_flag 均为 0。回答可用中文业务表述或给出这些标识，不要求固定格式。",
            "question": "最终回复是否识别出两条目标授权分别为人员授权与厂商远程维护授权，并准确说明两条记录操作前为已撤权但会话活跃、操作后仍为已撤权且会话已终止？",
            "tier": "important"
          },
          {
            "files": [
              "answer.md"
            ],
            "id": "operation_scope_reported",
            "method": "llm",
            "pass_anchor": "真实操作口径：仅对 ENT-2016-00417 与 ENT-2016-00503 将活跃会话清零；两条的 authorization_status 继续为 revoked，access_scope、其他授权及非目标机构不应被修改。最终回复需报告已执行的会话终止及范围守恒结论，不能只给出计划。",
            "question": "最终回复是否明确说明实际执行的是仅终止这两条已撤权授权的活跃会话，并确认未改变授权状态、访问范围或其他机构/其他授权？",
            "tier": "important"
          },
          {
            "fn": "check_non_target_preserved",
            "ground_truth": {
              "allowed_changes": [
                {
                  "column": "active_session_flag",
                  "entitlement_id": "ENT-2016-00417",
                  "from": 1,
                  "to": 0
                },
                {
                  "column": "active_session_flag",
                  "entitlement_id": "ENT-2016-00503",
                  "from": 1,
                  "to": 0
                }
              ]
            },
            "id": "non_target_preserved",
            "method": "rule",
            "question": "",
            "tier": "sanity"
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Weighted sum divided by all criterion weights; missing source weights are 1. Tiers are metadata, not implicit gates.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Keeps real state checks separate from narrative claims.",
          "Tests both the intended change and preservation of non-target authorizations."
        ],
        "risks": [
          "Two criteria are opaque executable checks; prose cannot replace running their original verifier.",
          "A fluent completion claim could be fabricated unless the state transition is observed."
        ],
        "must_hold": [
          "Terminate only the two identified sessions; leave revoked status, access scope and unrelated records intact.",
          "Identify workforce versus vendor and report observed before/after state."
        ],
        "may_vary": [
          "Different permitted action sequences or tool calls that yield the same authorized state.",
          "Chinese prose or explicit IDs for the before/after report, as the source anchor permits."
        ],
        "proposed_changes": [
          "Run state-delta assertions independently of final wording.",
          "Exercise two permitted action orders plus a no-op and an overbroad-change control."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: terminate workforce then vendor",
          "kind": "valid_alternative",
          "response": "核验两条目标授权后，先终止人员授权会话，再终止厂商会话；重新读取两条状态并核对其他授权未变。报告保留的 revoked 状态和 active_session_flag 从 1 到 0 的变化。",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Action ordering is incidental; completion requires actual execution."
        },
        {
          "label": "Sketch B: reverse action order",
          "kind": "valid_alternative",
          "response": "先处理厂商远程维护会话，再处理人员会话；逐条确认仍已撤权，并以操作前后差异记录证明访问范围和非目标记录未改变。",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "The reverse order can satisfy the same invariants if the environment permits it."
        },
        {
          "label": "Invalid: restore authorization",
          "kind": "invalid_control",
          "response": "为方便清理，我先恢复两条授权，扩大维护权限后再退出会话。",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Changes expressly forbidden authorization and scope state."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "proposed-auto-rubric-9d1d898ec58f88f43a7b358f78bac996",
      "title": "Gather panorama images without requiring one filesystem API",
      "source": {
        "dataset": "agentscope-ai/Auto-Rubric",
        "url": "https://huggingface.co/datasets/agentscope-ai/Auto-Rubric/blob/2a5e18a3a1d57367bb907cf7a40ce33219e29385/train.jsonl",
        "revision": "2a5e18a3a1d57367bb907cf7a40ce33219e29385",
        "split": "train",
        "row_id": "9d1d898ec58f88f43a7b358f78bac996",
        "row_index_in_file": 25,
        "source_file": "train.jsonl",
        "license": "Apache-2.0 annotations; underlying HelpSteer3 CC-BY-4.0",
        "pool": "proposed",
        "row_sha256": "2bc022f0cb0f2bfc0768f99a288d6fb7b9c09c5737eb246ede178c4588a8b26a",
        "attribution": "Source task and reference: agentscope-ai/Auto-Rubric. Editorial rubric drafts, reviews and response probes were added for this gallery; original source text remains in the raw record. Underlying conversations: NVIDIA HelpSteer3, CC-BY-4.0.",
        "links": [
          {
            "label": "Pinned Hugging Face source file",
            "url": "https://huggingface.co/datasets/agentscope-ai/Auto-Rubric/blob/2a5e18a3a1d57367bb907cf7a40ce33219e29385/train.jsonl"
          },
          {
            "label": "Underlying HelpSteer3 dataset",
            "url": "https://huggingface.co/datasets/nvidia/HelpSteer3"
          }
        ]
      },
      "taxonomy": {
        "pillar": "build_operate",
        "family": "terminal_data_pipelines",
        "leaf": "terminal_data_pipelines.file_artifact_processing",
        "confidence": "editorial_high",
        "reason": "Requires locating and copying filesystem artifacts with name transformation."
      },
      "language": "en",
      "prompt": "I have a folder containing many subfolders. Inside each subfolder, there is a photo named panorama, but the format may vary, like jpg, png or jpeg. Can you help me write a python script, which does the following things:\n1. Create a new folder named p3_artifacts\n2. Scan each subfolder, find the photo named with panorama, copy it to the p3_artifacts folder, and rename it with its subfolder's name",
      "context": [],
      "environment_note": "Self-contained text task; no external tools are required.",
      "rubric": {
        "markdown": "# Source-provided rubric\n\nSource text and weights are reproduced verbatim below. No grading anchors or aggregation semantics have been inferred.\n\n### Source criterion 1\n\n**Criterion:** A better answer should demonstrate correct implementation of file operations without runtime errors, handle variable file formats and case-insensitive filename matching, avoid hardcoded paths requiring user modification, and ensure reliable detection and renaming of target files according to the specified requirements. Answer 1 achieves this by using os.listdir() with case-insensitive checks, constructing paths dynamically without hardcoded values, and including necessary imports, while Answer 2 fails due to missing imports (e.g., glob), hardcoded paths, and unreliable file detection patterns.\n\n**Points:** Not supplied by source\n\n**Grading:** Not supplied by source.\n\n**Tags:** Not supplied by source.\n",
        "raw": {
          "rubrics": [
            "A better answer should demonstrate correct implementation of file operations without runtime errors, handle variable file formats and case-insensitive filename matching, avoid hardcoded paths requiring user modification, and ensure reliable detection and renaming of target files according to the specified requirements. Answer 1 achieves this by using os.listdir() with case-insensitive checks, constructing paths dynamically without hardcoded values, and including necessary imports, while Answer 2 fails due to missing imports (e.g., glob), hardcoded paths, and unreliable file detection patterns."
          ],
          "rubric_valid": "True",
          "preferred": "A",
          "source": "helpsteer3_preference"
        },
        "origin": "source-provided",
        "criterion_count": 1
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Source captures key copying and renaming requirements."
        ],
        "risks": [
          "Native rubric refers to Answer 1/Answer 2 and favors one implementation.",
          "No filesystem implementation was run for these review sketches.",
          "The prompt does not fully specify recursion depth or duplicate matches."
        ],
        "must_hold": [
          "Create the destination directory.",
          "Copy the requested panorama files and rename by containing subfolder."
        ],
        "may_vary": [
          "pathlib, os.scandir, os.listdir or glob.",
          "Command-line root path or clearly explained configuration.",
          "Documented treatment of ambiguous duplicate formats."
        ],
        "proposed_changes": [
          "Remove response-specific comparisons and method favoritism.",
          "Test output artifacts on fixtures instead of checking for os.listdir text."
        ],
        "revised_rubric_markdown": "# Proposed inclusive revision: Gather panorama images without requiring one filesystem API\n\n**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### Discover source images\n\n**ID:** c1\n\n**Criterion:** Scan the relevant subfolders and locate their panorama images in the stated varying formats. State a reasonable behavior for missing or multiple matches without silently losing files.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Correctness\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Copy and rename\n\n**ID:** c2\n\n**Criterion:** Create p3\\_artifacts, copy each selected image there, and use the containing subfolder name as the new basename while preserving the image format/extension.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Task fidelity\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n\n### Usable script\n\n**ID:** c3\n\n**Criterion:** Provide runnable Python with required imports and an understandable root-folder input. os/pathlib/glob and equivalent copy mechanisms are all acceptable; protect original files from destructive changes.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Fully satisfies this criterion, including an equivalent valid approach.\n- **0.5:** Partly satisfies it with a material omission but no contradictory central claim.\n- **0:** Does not satisfy it, or contradicts the criterion.\n\n**Tags:** \\[\"Executability\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\"task\", \"source\\_reference\"\\]\n"
      },
      "probes": [
        {
          "label": "Pathlib route",
          "kind": "valid_alternative",
          "response": "Use Path(root).iterdir() for direct child directories, filter images whose stem is panorama and suffix is jpg/jpeg/png, create p3_artifacts, and shutil.copy2 to <folder-name><suffix>. Explicitly skip the destination directory and report duplicate matches.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "Uses a different library style while satisfying artifact requirements.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "os.scandir route",
          "kind": "valid_alternative",
          "response": "Scan child directories using os.scandir, match supported panorama filenames, create p3_artifacts with os.makedirs(exist_ok=True), and copy using shutil.copyfile to the renamed destination.",
          "is_full_response": false,
          "expected": "Should satisfy the task under the stated assumptions.",
          "verification": "unexecuted_sketch",
          "why": "Another valid implementation route, conditional on correct collision and error handling.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        },
        {
          "label": "Rename in place",
          "kind": "invalid_control",
          "response": "For every source image, use os.rename to move it directly into p3_artifacts.",
          "is_full_response": false,
          "expected": "Should fail at least one task-critical criterion.",
          "verification": "unexecuted_sketch",
          "why": "Moving deletes the original from its folder, whereas the user asked to copy.",
          "verification_detail": "Editorial review only; no external grader or runtime validation."
        }
      ],
      "source_record": {
        "unique_id": "9d1d898ec58f88f43a7b358f78bac996",
        "source": "helpsteer3_preference",
        "rubrics": [
          "A better answer should demonstrate correct implementation of file operations without runtime errors, handle variable file formats and case-insensitive filename matching, avoid hardcoded paths requiring user modification, and ensure reliable detection and renaming of target files according to the specified requirements. Answer 1 achieves this by using os.listdir() with case-insensitive checks, constructing paths dynamically without hardcoded values, and including necessary imports, while Answer 2 fails due to missing imports (e.g., glob), hardcoded paths, and unreliable file detection patterns."
        ],
        "rubric_valid": "True",
        "rubric_epoch": "1",
        "input": [
          {
            "role": "user",
            "content": "I have a folder containing many subfolders. Inside each subfolder, there is a photo named panorama, but the format may vary, like jpg, png or jpeg. Can you help me write a python script, which does the following things:\n1. Create a new folder named p3_artifacts\n2. Scan each subfolder, find the photo named with panorama, copy it to the p3_artifacts folder, and rename it with its subfolder's name"
          }
        ],
        "output": [
          {
            "answer": {
              "role": "assistant",
              "content": "```python\nimport os\nimport shutil\n\n# Create the destination folder\nos.makedirs(\"p3_artifacts\", exist_ok=True)\n\n# Iterate through subfolders\nfor subfolder in os.listdir(\".\"):\n    if os.path.isdir(subfolder):\n        # Find the panorama photo in the subfolder\n        for filename in os.listdir(subfolder):\n            if \"panorama\" in filename.lower():\n                # Construct the source and destination file paths\n                source_path = os.path.join(subfolder, filename)\n                destination_path = os.path.join(\"p3_artifacts\", subfolder + \"_\" + filename)\n\n                # Copy and rename the file\n                shutil.copy(source_path, destination_path)\n                print(f\"Copied '{filename}' from '{subfolder}' to '{destination_path}'\")\n                break  # Move to the next subfolder after finding one panorama photo\n\nprint(\"Finished processing all subfolders.\")\n```\n\n**Explanation:**\n\n1. **Import necessary modules:**\n   - `os`: For interacting with the file system (listing directories, creating folders, etc.).\n   - `shutil`: For copying files.\n\n2. **Create the destination folder:**\n   - `os.makedirs(\"p3_artifacts\", exist_ok=True)` creates a folder named \"p3_artifacts\" if it doesn't exist. The `exist_ok=True` prevents an error if the folder already exists.\n\n3. **Iterate through subfolders:**\n   - `os.listdir(\".\")` lists all items in the current directory.\n   - The `if os.path.isdir(subfolder):` checks if the item is a directory (subfolder).\n\n4. **Find the panorama photo:**\n   - `for filename in os.listdir(subfolder):` iterates through files in the subfolder.\n   - `if \"panorama\" in filename.lower():` checks if the filename (converted to lowercase) contains \"panorama\".\n\n5. **Construct file paths:**\n   - `source_path` is the full path to the panorama photo in the subfolder.\n   - `destination_path` is the full path to the destination file in the \"p3_artifacts\" folder, with the subfolder name prepended to the filename.\n\n6. **Copy and rename:**\n   - `shutil.copy(source_path, destination_path)` copies the file.\n   - A message is printed to confirm the copy.\n   - `break` exits the inner loop after finding and copying one panorama photo per subfolder.\n\n7. **Print completion message:**\n   - After processing all subfolders, a message indicates the script has finished.\n\n**How to use:**\n\n1. Save the code as a Python file (e.g., `panorama_copy.py`).\n2. Place the script in the folder containing your subfolders.\n3. Run the script from the command line: `python panorama_copy.py`\n\nThe script will create the \"p3_artifacts\" folder and copy the panorama photos from each subfolder, renaming them accordingly.",
              "label": {
                "is_preferred": true,
                "preference": "chosen",
                "response_type": "A"
              }
            }
          },
          {
            "answer": {
              "role": "assistant",
              "content": "Sure, here is a Python script that should accomplish what you're asking for:\n```python\nimport os\nimport shutil\n\n# Create the new folder\nnew_folder = \"p3_artifacts\"\nos.makedirs(new_folder, exist_ok=True)\n\n# Define the base directory where the subfolders are located\nbase_dir = \"/path/to/your/folder\"\n\n# Iterate through each subfolder in the base directory\nfor root, dirs, files in os.walk(base_dir):\n    for dir in dirs:\n        subfolder_path = os.path.join(root, dir)\n        panorama_path = os.path.join(subfolder_path, \"panorama.*\")\n        file_path = glob.glob(panorama_path)[0]\n        if file_path:\n            # Copy the file to the new folder and rename it\n            filename = os.path.basename(file_path)\n            new_path = os.path.join(new_folder, f\"{dir}_{filename}\")\n            shutil.copy(file_path, new_path)\n```\nHere's how the script works:\n\n1. It creates a new folder named \"p3\\_artifacts\" in the current working directory.\n2. It defines the base directory where the subfolders are located.\n3. It uses the `os.walk()` function to iterate through each subfolder in the base directory.\n4. For each subfolder, it searches for a file named \"panorama\" with any of the supported formats (jpg, png, or jpeg) using the `glob.glob()` function.\n5. If a matching file is found, it copies the file to the \"p3\\_artifacts\" folder using the `shutil.copy()` function and renames it with the subfolder's name and the original filename.\n\nNote that you'll need to replace `\"/path/to/your/folder\"` with the actual path to your folder containing the subfolders.",
              "label": {
                "is_preferred": false,
                "preference": "rejected",
                "response_type": "B"
              }
            }
          }
        ],
        "preferred": "A",
        "domain": "code",
        "language": "python"
      },
      "verification": {
        "status": "editorial_review",
        "method": "Initial editorial review; no model rubric judge was run.",
        "checks": [],
        "limitations": [
          "Claims about full response quality have not been independently graded.",
          "Sketches are candidate response strategies, not verified completed solutions."
        ]
      },
      "admission": "gallery_only_not_accepted_training_data"
    },
    {
      "id": "current:researchrubrics:6847465956a0f6376a605424",
      "title": "Study locations: preferences versus invented completeness",
      "source": {
        "dataset": "ScaleAI/researchrubrics",
        "source_name": "researchrubrics",
        "url": "https://huggingface.co/datasets/ScaleAI/researchrubrics",
        "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
        "original_inventory_revision": "local-sha256:ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "revision_kind": "verified_huggingface_commit",
        "revision_evidence": "Pinned processed_data.jsonl SHA256 exactly matches the original inventory source file",
        "split": "processed_data.jsonl (no split name declared)",
        "row_id": "6847465956a0f6376a605424",
        "license": "MIT",
        "pool": "current",
        "source_content_sha256": "ea2023d03953b70ba4a2f7f1fe490b434f18cec26c4fdb48acc0e3edf3d8edfb",
        "source_record_fingerprint": "54bcb61919239b2acc28ea0e071bef2607be827129c811ade9629a1964492ddd",
        "source_row_index": 10,
        "canonical_inventory_sha256": "7e795730648172e3b9a2d2e1bdb21bd10941b851a2d408294f9d57b8102b3ae5",
        "expert_authored_claim_from_import": true,
        "synthetic_claim_from_import": false,
        "attribution": "Source task and rubric: researchrubrics; Markdown rendering, editorial review, display tags and probes added for this gallery."
      },
      "taxonomy": {
        "pillar": "communication_adaptation",
        "family": "user_preferences",
        "leaf": "user_preferences.personalized_selection_ranking",
        "confidence": "editorial_provisional",
        "reason": "Recommendations must satisfy this user's explicit cost, parking, study-space, hours and travel constraints."
      },
      "language": "en",
      "prompt": "I am a high schooler in Cupertino. It is AP season and I'm trying to study for my tests. Can you give me 10 recommendations for places that open until at least 10pm that are good for studying (fast wifi, sufficient table space to do practice tests, good study environment, etc)? I would prefer not having to spend money on drinks/food in cafes but have a $10 budget if I must, and absolutely refuse to spend money on parking/fees. I am only open to places that are at most at 20 minute drive from me.",
      "context": [],
      "environment_note": "No source retrieval or independent grading was run. The canonical inventory still marks this task blocked pending environment and validation. Self-contained editorial responses, when supplied, remain candidates; other probes are visibly incomplete sketches.",
      "rubric": {
        "markdown": "**Format:** rubric-markdown-v1\n\n**Scoring:** positive_weights\n\nFor non-gate criteria, sum Points × activation and divide by the sum of positive Points.\nA positive finite denominator is required. Gate criteria are excluded from aggregation; if any gate activation is below 0.999, the score is 0. Scores are not clipped.\n\n### The response contains exactly 10 distinct study-location recommendations.\n\n**ID:** c1\n\n**Criterion:** The response contains exactly 10 distinct study-location recommendations.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Every recommended location name is accompanied by either a full street address or, at minimum, a clearly identified city/neighborhood (e.g., “Philz Coffee – Sunnyvale”).\n\n**ID:** c2\n\n**Criterion:** Every recommended location name is accompanied by either a full street address or, at minimum, a clearly identified city/neighborhood (e.g., “Philz Coffee – Sunnyvale”).\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### All recommended locations are situated in a city that is within a typical 20-minute drive from Cupertino during non-rush hours (Cupertino, Sunnyvale, Santa Clara, Mountain View, Los Altos, Saratoga, West San Jose, etc)\n\n**ID:** c3\n\n**Criterion:** All recommended locations are situated in a city that is within a typical 20-minute drive from Cupertino during non-rush hours (Cupertino, Sunnyvale, Santa Clara, Mountain View, Los Altos, Saratoga, West San Jose, etc)\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### For every recommendation, the response states that the closing time is 10:00 PM or later on weekdays (and specifies the closing time).\n\n**ID:** c4\n\n**Criterion:** For every recommendation, the response states that the closing time is 10:00 PM or later on weekdays (and specifies the closing time).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Some locations that close before 10:00 p.m. on weekdays are listed as meeting the ≥10 p.m. requirement (e.g., Cupertino Library, which closes at 9 p.m.).\n\n**ID:** c5\n\n**Criterion:** Some locations that close before 10:00 p.m. on weekdays are listed as meeting the ≥10 p.m. requirement (e.g., Cupertino Library, which closes at 9 p.m.).\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### For each location, the response explicitly mentions the availability of free, reliable Wi-Fi (using language such as “free Wi-Fi” or “fast Wi-Fi available”).\n\n**ID:** c6\n\n**Criterion:** For each location, the response explicitly mentions the availability of free, reliable Wi-Fi (using language such as “free Wi-Fi” or “fast Wi-Fi available”).\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### For each location, the response explicitly mentions sufficient table/desk space or seating appropriate for studying. Example: \n\n**ID:** c7\n\n**Criterion:** For each location, the response explicitly mentions sufficient table/desk space or seating appropriate for studying. Example: <br>\"Cupertino Library has large communal tables and quiet study carrels ideal for spreading out AP practice materials.\"\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### For every location, the response either confirms unlimited free parking or states that no parking fees apply.\n\n**ID:** c8\n\n**Criterion:** For every location, the response either confirms unlimited free parking or states that no parking fees apply.\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Whenever a café or restaurant is suggested, the response suggests items that can be obtained for ≤ $10 in total.\n\n**ID:** c9\n\n**Criterion:** Whenever a café or restaurant is suggested, the response suggests items that can be obtained for ≤ $10 in total.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response is structured as a clear numbered/bulleted list or table so that a reader can quickly scan the 10 options.\n\n**ID:** c10\n\n**Criterion:** The response is structured as a clear numbered/bulleted list or table so that a reader can quickly scan the 10 options.\n\n**Points:** +4\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response includes a brief note advising the user to verify operating hours before visiting (demonstrating awareness of possible schedule changes).\n\n**ID:** c11\n\n**Criterion:** The response includes a brief note advising the user to verify operating hours before visiting (demonstrating awareness of possible schedule changes).\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response does not recommend purely outdoor locations that lack tables or reliable shelter (e.g., public parks with only benches), unless they clearly state tables and weather-protected seating are available.\n\n**ID:** c12\n\n**Criterion:** The response does not recommend purely outdoor locations that lack tables or reliable shelter (e.g., public parks with only benches), unless they clearly state tables and weather-protected seating are available.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### For each library or public facility included, the response confirms that non-residents may enter without special membership/fees. Example: \"Santa Clara City Library allows all California residents to enter and use study spaces without a membership fee (verified on the library’s official visitor policy page).\"\n\n**ID:** c13\n\n**Criterion:** For each library or public facility included, the response confirms that non-residents may enter without special membership/fees. Example: \"Santa Clara City Library allows all California residents to enter and use study spaces without a membership fee (verified on the library’s official visitor policy page).\"\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The response should mention if any location is known for stable power outlets, especially for laptop-based studying.\n\n**ID:** c14\n\n**Criterion:** The response should mention if any location is known for stable power outlets, especially for laptop-based studying.\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Any parking references should note if the location tends to have full or difficult parking. Eample: \"Main Street Cupertino’s public parking garage is free but often fills up after 7 PM, so arriving early is recommended.\"\n\n**ID:** c15\n\n**Criterion:** Any parking references should note if the location tends to have full or difficult parking. Eample: \"Main Street Cupertino’s public parking garage is free but often fills up after 7 PM, so arriving early is recommended.\"\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Response notes that public study spaces will be extra crowded during AP season as everyone is trying to study as well\n\n**ID:** c16\n\n**Criterion:** Response notes that public study spaces will be extra crowded during AP season as everyone is trying to study as well\n\n**Points:** +2\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### All locations are accurate (i.e., still open, not under renovations, name of place is correct, address is correct)\n\n**ID:** c17\n\n**Criterion:** All locations are accurate (i.e., still open, not under renovations, name of place is correct, address is correct)\n\n**Points:** +5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### The recommendations include private office buildings or coworking spaces with restricted or employee-only entry. E.g., WeWork Cupertino which requires a paid membership or company access.\n\n**ID:** c18\n\n**Criterion:** The recommendations include private office buildings or coworking spaces with restricted or employee-only entry. E.g., WeWork Cupertino which requires a paid membership or company access.\n\n**Points:** -5\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### If recommending a location with a minimum stay limit or a time limit for seating (common in some boba cafés), the response should disclose it. Example: \"Fantasia Coffee &amp; Tea in Cupertino has a posted 90-minute seating limit during evenings, so it’s better for quick review sessions rather than full-length AP practice tests.\"\n\n**ID:** c19\n\n**Criterion:** If recommending a location with a minimum stay limit or a time limit for seating (common in some boba cafés), the response should disclose it. Example: \"Fantasia Coffee &amp; Tea in Cupertino has a posted 90-minute seating limit during evenings, so it’s better for quick review sessions rather than full-length AP practice tests.\"\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### If a place is louder/more bustling/less favorable for studying, response should always acknowledge this. Example: \"Main Street Cupertino Plaza has outdoor seating and stays open late, but it’s usually busy with restaurant traffic and live music on weekends - so not ideal for quiet studying.\"\n\n**ID:** c20\n\n**Criterion:** If a place is louder/more bustling/less favorable for studying, response should always acknowledge this. Example: \"Main Street Cupertino Plaza has outdoor seating and stays open late, but it’s usually busy with restaurant traffic and live music on weekends - so not ideal for quiet studying.\"\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n\n### Whenever a café or restaurant is suggested, the response states whether purchase is optional. Example: \"Panera Bread on Stevens Creek Boulevard allows customers to sit and use Wi-Fi without constant purchases, though a small drink (~$4) is expected if you stay for several hours.\"\n\n**ID:** c21\n\n**Criterion:** Whenever a café or restaurant is suggested, the response states whether purchase is optional. Example: \"Panera Bread on Stevens Creek Boulevard allows customers to sit and use Wi-Fi without constant purchases, though a small drink (~$4) is expected if you stay for several hours.\"\n\n**Points:** +3\n\n**Grading:**\n\n- **1:** Criterion is satisfied\n- **0.5:** Criterion is partially satisfied\n- **0:** Criterion is not satisfied\n\n**Tags:** \\[\"User preferences\", \"en\", \"Source criterion\"\\]\n\n**Method:** llm\n\n**Gate:** false\n\n**Status:** provisional\n\n**Evidence:** \\[\\]\n",
        "raw": {
          "criteria": [
            {
              "criterion": "The response contains exactly 10 distinct study-location recommendations.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c1",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "Every recommended location name is accompanied by either a full street address or, at minimum, a clearly identified city/neighborhood (e.g., “Philz Coffee – Sunnyvale”).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c2",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "All recommended locations are situated in a city that is within a typical 20-minute drive from Cupertino during non-rush hours (Cupertino, Sunnyvale, Santa Clara, Mountain View, Los Altos, Saratoga, West San Jose, etc)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c3",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "For every recommendation, the response states that the closing time is 10:00 PM or later on weekdays (and specifies the closing time).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c4",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "Some locations that close before 10:00 p.m. on weekdays are listed as meeting the ≥10 p.m. requirement (e.g., Cupertino Library, which closes at 9 p.m.).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c5",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "For each location, the response explicitly mentions the availability of free, reliable Wi-Fi (using language such as “free Wi-Fi” or “fast Wi-Fi available”).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c6",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "For each location, the response explicitly mentions sufficient table/desk space or seating appropriate for studying. Example: \n\"Cupertino Library has large communal tables and quiet study carrels ideal for spreading out AP practice materials.\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c7",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "For every location, the response either confirms unlimited free parking or states that no parking fees apply.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c8",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "Whenever a café or restaurant is suggested, the response suggests items that can be obtained for ≤ $10 in total.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c9",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response is structured as a clear numbered/bulleted list or table so that a reader can quickly scan the 10 options.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c10",
              "method": "llm",
              "status": "provisional",
              "weight": 4.0
            },
            {
              "criterion": "The response includes a brief note advising the user to verify operating hours before visiting (demonstrating awareness of possible schedule changes).",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c11",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "The response does not recommend purely outdoor locations that lack tables or reliable shelter (e.g., public parks with only benches), unless they clearly state tables and weather-protected seating are available.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c12",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "For each library or public facility included, the response confirms that non-residents may enter without special membership/fees. Example: \"Santa Clara City Library allows all California residents to enter and use study spaces without a membership fee (verified on the library’s official visitor policy page).\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c13",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "The response should mention if any location is known for stable power outlets, especially for laptop-based studying.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c14",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "Any parking references should note if the location tends to have full or difficult parking. Eample: \"Main Street Cupertino’s public parking garage is free but often fills up after 7 PM, so arriving early is recommended.\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c15",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "Response notes that public study spaces will be extra crowded during AP season as everyone is trying to study as well",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c16",
              "method": "llm",
              "status": "provisional",
              "weight": 2.0
            },
            {
              "criterion": "All locations are accurate (i.e., still open, not under renovations, name of place is correct, address is correct)",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c17",
              "method": "llm",
              "status": "provisional",
              "weight": 5.0
            },
            {
              "criterion": "The recommendations include private office buildings or coworking spaces with restricted or employee-only entry. E.g., WeWork Cupertino which requires a paid membership or company access.",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c18",
              "method": "llm",
              "status": "provisional",
              "weight": -5.0
            },
            {
              "criterion": "If recommending a location with a minimum stay limit or a time limit for seating (common in some boba cafés), the response should disclose it. Example: \"Fantasia Coffee & Tea in Cupertino has a posted 90-minute seating limit during evenings, so it’s better for quick review sessions rather than full-length AP practice tests.\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c19",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "If a place is louder/more bustling/less favorable for studying, response should always acknowledge this. Example: \"Main Street Cupertino Plaza has outdoor seating and stays open late, but it’s usually busy with restaurant traffic and live music on weekends - so not ideal for quiet studying.\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c20",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            },
            {
              "criterion": "Whenever a café or restaurant is suggested, the response states whether purchase is optional. Example: \"Panera Bread on Stevens Creek Boulevard allows customers to sit and use Wi-Fi without constant purchases, though a small drink (~$4) is expected if you stay for several hours.\"",
              "evidence_refs": [],
              "gate": false,
              "grading_anchors": {
                "0": "Criterion is not satisfied",
                "0.5": "Criterion is partially satisfied",
                "1": "Criterion is satisfied"
              },
              "id": "c21",
              "method": "llm",
              "status": "provisional",
              "weight": 3.0
            }
          ],
          "scoring": "positive_weights",
          "title": ""
        },
        "origin": "source-provided",
        "criterion_count": 21,
        "source_original_items": [
          {
            "axis": "Instruction Following",
            "criterion": "The response contains exactly 10 distinct study-location recommendations.",
            "weight": 5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "Every recommended location name is accompanied by either a full street address or, at minimum, a clearly identified city/neighborhood (e.g., “Philz Coffee – Sunnyvale”).",
            "weight": 3.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "All recommended locations are situated in a city that is within a typical 20-minute drive from Cupertino during non-rush hours (Cupertino, Sunnyvale, Santa Clara, Mountain View, Los Altos, Saratoga, West San Jose, etc)",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "For every recommendation, the response states that the closing time is 10:00 PM or later on weekdays (and specifies the closing time).",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "Some locations that close before 10:00 p.m. on weekdays are listed as meeting the ≥10 p.m. requirement (e.g., Cupertino Library, which closes at 9 p.m.).",
            "weight": -5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "For each location, the response explicitly mentions the availability of free, reliable Wi-Fi (using language such as “free Wi-Fi” or “fast Wi-Fi available”).",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "For each location, the response explicitly mentions sufficient table/desk space or seating appropriate for studying. Example: \n\"Cupertino Library has large communal tables and quiet study carrels ideal for spreading out AP practice materials.\"",
            "weight": 5.0
          },
          {
            "axis": "Instruction Following",
            "criterion": "For every location, the response either confirms unlimited free parking or states that no parking fees apply.",
            "weight": 5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "Whenever a café or restaurant is suggested, the response suggests items that can be obtained for ≤ $10 in total.",
            "weight": 3.0
          },
          {
            "axis": "Communication Quality",
            "criterion": "The response is structured as a clear numbered/bulleted list or table so that a reader can quickly scan the 10 options.",
            "weight": 4.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response includes a brief note advising the user to verify operating hours before visiting (demonstrating awareness of possible schedule changes).",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response does not recommend purely outdoor locations that lack tables or reliable shelter (e.g., public parks with only benches), unless they clearly state tables and weather-protected seating are available.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "For each library or public facility included, the response confirms that non-residents may enter without special membership/fees. Example: \"Santa Clara City Library allows all California residents to enter and use study spaces without a membership fee (verified on the library’s official visitor policy page).\"",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The response should mention if any location is known for stable power outlets, especially for laptop-based studying.",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "Any parking references should note if the location tends to have full or difficult parking. Eample: \"Main Street Cupertino’s public parking garage is free but often fills up after 7 PM, so arriving early is recommended.\"",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "Response notes that public study spaces will be extra crowded during AP season as everyone is trying to study as well",
            "weight": 2.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "All locations are accurate (i.e., still open, not under renovations, name of place is correct, address is correct)",
            "weight": 5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "The recommendations include private office buildings or coworking spaces with restricted or employee-only entry. E.g., WeWork Cupertino which requires a paid membership or company access.",
            "weight": -5.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "If recommending a location with a minimum stay limit or a time limit for seating (common in some boba cafés), the response should disclose it. Example: \"Fantasia Coffee & Tea in Cupertino has a posted 90-minute seating limit during evenings, so it’s better for quick review sessions rather than full-length AP practice tests.\"",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "If a place is louder/more bustling/less favorable for studying, response should always acknowledge this. Example: \"Main Street Cupertino Plaza has outdoor seating and stays open late, but it’s usually busy with restaurant traffic and live music on weekends - so not ideal for quiet studying.\"",
            "weight": 3.0
          },
          {
            "axis": "Implicit Criteria",
            "criterion": "Whenever a café or restaurant is suggested, the response states whether purchase is optional. Example: \"Panera Bread on Stevens Creek Boulevard allows customers to sit and use Wi-Fi without constant purchases, though a small drink (~$4) is expected if you stay for several hours.\"",
            "weight": 3.0
          }
        ],
        "canonicalization_note": "Markdown preserves the current inventory's canonical criterion text, points, anchors and gates. This is an adapted source rubric, not a newly audited training target. Original source items are retained separately; titles/tags are gallery presentation metadata.",
        "source_scoring_note": "Source score semantics still require validation before training acceptance.",
        "source_category_groups": {}
      },
      "review": {
        "status": "initial_editorial_review",
        "strengths": [
          "Separates free parking, entry, purchase requirements, hours, workspace and noise.",
          "Rejects restricted-entry spaces presented as public recommendations."
        ],
        "risks": [
          "Exactly ten recommendations may be impossible under all constraints; a model could invent hours or access to fill the quota.",
          "A city-level 20-minute approximation does not verify travel time from an unspecified starting point.",
          "Current hours and policies require retrieval; no venues are verified in these sketches."
        ],
        "must_hold": [
          "Do not misstate opening hours, access, parking fees, purchase requirements or study facilities.",
          "Respect the $10 ceiling, free-parking requirement and travel limit, or clearly identify what cannot be verified."
        ],
        "may_vary": [
          "Any genuinely qualifying locations and ranking that respects hard constraints.",
          "An honest shortfall plus a targeted clarification when ten verified options cannot be found."
        ],
        "proposed_changes": [
          "Do not punish an evidence-backed infeasibility finding in order to reward an invented list of ten.",
          "Separate verified fits from conditional candidates; ask for starting area/date only when needed.",
          "Treat reliability of the factual claims as a gate before count and presentation points."
        ]
      },
      "probes": [
        {
          "label": "Sketch A: ten verified fits if available",
          "kind": "valid_alternative",
          "response": "Produce a table of independently verified places with dated hours, access, parking, purchase policy, workspace, and travel estimates from a stated Cupertino starting point. Rank free options first and disclose noise or seating limits.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "A valid path if ten actual options can be demonstrated; the gallery does not assert they exist."
        },
        {
          "label": "Sketch B: transparent infeasibility",
          "kind": "valid_alternative",
          "response": "After checking the relevant policies and hours, report only the verified qualifying places if fewer than ten exist. Explain which constraints rule out the others and ask whether the user would relax distance or closing time; do not include nonqualifiers as matches.",
          "is_full_response": false,
          "expected": "Illustrative response sketch only; complete evidence or execution is still required.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Preserves truth and the hard preferences instead of fabricating completeness."
        },
        {
          "label": "Invalid: manufacture a late-closing library",
          "kind": "invalid_control",
          "response": "Cupertino Library is open until midnight every weekday, parking is always available, and no verification is needed.",
          "is_full_response": false,
          "expected": "Reject the identified violation; this is not a scored model run.",
          "verification": "Editorial analysis only; no independent grader, source retrieval, or environment execution.",
          "why": "Asserts unverified specific hours and availability to satisfy the requested form."
        }
      ],
      "admission": "gallery_only_not_accepted_training_data"
    }
  ]
}