"""Checks for educational code and arithmetic, not a utility model validation.""" import json import math import subprocess import sys import tempfile import unittest import xml.etree.ElementTree as ET from dataclasses import replace from pathlib import Path from demo import (calibration_bins, chart_svg, error_slices, evaluate, features, fit, grouped_split, make_records, predict, run, sigmoid) class SyntheticCompanionTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.records = make_records() cls.train, cls.test = grouped_split(cls.records) cls.weights = fit(cls.train) cls.report = run() def test_generation_is_deterministic(self): self.assertEqual(self.records, make_records()) self.assertNotEqual(self.records, make_records(seed=99)) def test_split_is_input_order_independent(self): self.assertEqual((self.train, self.test), grouped_split(list(reversed(self.records)))) def test_groups_and_properties_do_not_leak(self): for attribute in ("group_id", "property_id"): self.assertFalse({getattr(row, attribute) for row in self.train} & {getattr(row, attribute) for row in self.test}) self.assertEqual(len(self.train) + len(self.test), 800) self.assertEqual(len(self.test), 200) self.assertEqual(self.report["split"]["train_groups"], 120) self.assertEqual(self.report["split"]["test_groups"], 40) def test_duplicate_property_rows_are_rejected(self): for duplicate in (self.records[0], replace(self.records[0], group_id="other")): with self.assertRaises(ValueError): grouped_split(self.records + [duplicate]) def test_labels_and_identifiers_are_not_features(self): row = self.records[0] changed = replace(row, target=1 - row.target, property_id="other", group_id="other") self.assertEqual(features(row), features(changed)) def test_keyword_is_noisy_evidence_not_a_label(self): self.assertTrue(any(row.keyword_evidence != row.target for row in self.records)) def test_missing_age_is_explicit_and_finite(self): values = features(replace(self.records[0], installation_age=None)) self.assertEqual(values[-1], 1.0) self.assertEqual(values[1], 0.0) self.assertTrue(all(math.isfinite(value) for value in values)) def test_sigmoid_is_stable(self): self.assertEqual(sigmoid(1000), 1.0) self.assertEqual(sigmoid(-1000), 0.0) def test_fit_is_repeatable_and_excludes_test_targets(self): self.assertEqual(self.weights, fit(self.train)) test_ids = {row.property_id for row in self.test} changed = [replace(row, target=1 - row.target) if row.property_id in test_ids else row for row in self.records] changed_train, changed_test = grouped_split(changed) self.assertEqual(changed_train, self.train) self.assertEqual({row.property_id for row in changed_test}, test_ids) self.assertEqual(self.weights, fit(changed_train)) def test_metrics_match_hand_calculation(self): metrics = evaluate([0, 1], [0.25, 0.75]) self.assertAlmostEqual(metrics["brier_score"], 0.0625) self.assertAlmostEqual(metrics["log_loss"], -math.log(0.75)) self.assertEqual(metrics["accuracy"], 1) def test_confusion_arithmetic_and_threshold_boundary(self): metrics = evaluate([0, 1, 0, 1], [0.1, 0.8, 0.5, 0.2]) self.assertEqual(metrics["confusion"], {"true_negative": 1, "true_positive": 1, "false_negative": 1, "false_positive": 1}) self.assertEqual(metrics["accuracy"], 0.5) def test_reliability_includes_one_and_handles_empty_bins(self): bins = calibration_bins([0, 1, 0, 1], [0, 0.2, 0.99, 1.0]) self.assertEqual([item["count"] for item in bins], [1, 1, 0, 0, 2]) self.assertIsNone(bins[2]["observed_fraction"]) self.assertIsNone(bins[2]["mean_prediction"]) self.assertAlmostEqual(bins[4]["mean_prediction"], 0.995) self.assertEqual(bins[4]["observed_fraction"], 0.5) def test_all_reliability_records_are_counted_once(self): bins = self.report["reliability_bins"] self.assertEqual(sum(item["count"] for item in bins), 200) self.assertAlmostEqual(sum(item["count"] * item["observed_fraction"] for item in bins), self.report["logistic_model"]["positive_labels"]) def test_baseline_is_training_prevalence_only(self): prevalence = sum(row.target for row in self.train) / len(self.train) self.assertEqual(self.report["training_prevalence"], prevalence) expected = evaluate([row.target for row in self.test], [prevalence] * len(self.test)) self.assertEqual(self.report["training_prevalence_baseline"], expected) def test_intentionally_learnable_signal(self): learned = self.report["logistic_model"] baseline = self.report["training_prevalence_baseline"] self.assertLess(learned["log_loss"], baseline["log_loss"]) self.assertLess(learned["brier_score"], baseline["brier_score"]) def test_error_slice_partitions_and_confusion_totals(self): slices = self.report["error_slices"] for pair in (slices[:2], slices[2:]): self.assertEqual(sum(item["metrics"]["n"] for item in pair), 200) for key, value in self.report["logistic_model"]["confusion"].items(): self.assertEqual(sum(item["metrics"]["confusion"][key] for item in pair), value) empty = error_slices([replace(self.test[0], installation_age=None)], [0.5]) self.assertIsNone(empty[0]["metrics"]) def test_report_is_deterministic_and_presentation_complete(self): self.assertEqual(self.report, run()) for key in ("title", "disclosure", "metrics", "tables", "notes"): self.assertTrue(self.report[key]) for table in self.report["tables"]: self.assertTrue(all(len(row) == len(table["headers"]) for row in table["rows"])) json.dumps(self.report, allow_nan=False) self.assertIn("not the original", self.report["disclosure"]) def test_svg_is_valid_and_values_come_from_report(self): svg = chart_svg(self.report) root = ET.fromstring(svg) self.assertEqual(root.attrib["role"], "img") self.assertIn(f'{self.report["logistic_model"]["brier_score"]:.4f}', svg) self.assertIn(f'{self.report["training_prevalence_baseline"]["log_loss"]:.4f}', svg) self.assertIn("not field-validated", svg) self.assertNotIn("