hpr_validate/
committed.rs

1//! The committed reports, held to a run and to the accepted census: the check `cargo xtask
2//! validate --check` makes.
3//!
4//! [`check`] takes a run of every case ([`crate::run_lock`]) and fails unless
5//!
6//! - every scored metric is within its tolerance;
7//! - the run reproduces the committed report, `validation/reports/latest.{md,json}`, to the
8//!   digits the platforms share ([`Report::reproduces`]); and
9//! - the committed reports hold to the accepted census, `validation/reports/census.json`, and
10//!   every file written from it is what it writes ([`check_census`]; the census's decision record,
11//!   [ADR-084][adr-084]).
12//!
13//! The census is checked whether or not the run reproduces, so a report that fails both says so.
14//!
15//! [adr-084]: https://github.com/nrdptel/hpr-sim/blob/main/docs/DECISIONS.md#adr-084-the-accuracy-census-the-reports-numbers-held-to-the-ones-accepted-2026-09-26
16
17use std::path::{Path, PathBuf};
18
19use serde_json::Value;
20
21use crate::census::{self, Accepted, Census, Change, Reports};
22use crate::real_flight::RealFlightReport;
23use crate::report::Report;
24use crate::run::ValidateError;
25
26/// The harness's committed report, JSON.
27pub const LATEST_JSON: &str = "validation/reports/latest.json";
28/// The harness's committed report, Markdown.
29pub const LATEST_MD: &str = "validation/reports/latest.md";
30/// The accepted census.
31pub const CENSUS_JSON: &str = "validation/reports/census.json";
32
33/// What to do when the accepted census is missing: the hint that ends that refusal's message.
34pub const CENSUS_MISSING_HINT: &str = "run `cargo xtask census --accept --reason <why>`";
35
36/// What to do when the committed report is not this run's.
37pub const REPORT_HINT: &str =
38    "run `cargo xtask validate` and commit validation/reports/latest.{md,json}";
39/// Its page.
40pub const CENSUS_MD: &str = "validation/reports/census.md";
41/// Its page on GitHub, which the tables link to: the README and the site read it at one address.
42pub const CENSUS_URL: &str =
43    "https://github.com/nrdptel/hpr-sim/blob/main/validation/reports/census.md";
44/// Where the badges go.
45pub const BADGES: &str = "docs/images";
46/// The files that show the census table, between [`BEGIN`] and [`END`].
47pub const TABLES: [&str; 2] = ["README.md", "docs/accuracy.md"];
48/// The line that opens the census table in each of [`TABLES`].
49pub const BEGIN: &str = "<!-- census: written by `cargo xtask census --accept` from \
50                         validation/reports/census.json; do not edit -->";
51/// The line that closes it.
52pub const END: &str = "<!-- census: end -->";
53
54/// The committed reports against the accepted census.
55#[derive(Debug, Clone, PartialEq)]
56pub struct CensusCheck {
57    /// The accepted census's rows.
58    pub rows: usize,
59    /// The rows that differ from it.
60    pub changes: Vec<Change>,
61    /// The files written from it whose committed text isn't what it writes, each with why.
62    pub stale: Vec<String>,
63}
64
65impl CensusCheck {
66    /// Whether nothing differs and nothing is stale.
67    pub fn holds(&self) -> bool {
68        self.changes.is_empty() && self.stale.is_empty()
69    }
70
71    /// What `cargo xtask census --check` prints: a line per change and per stale file, and a
72    /// summary.
73    pub fn lines(&self) -> Vec<String> {
74        if self.holds() {
75            return vec![format!(
76                "census: the committed reports hold to the accepted census ({} rows)",
77                self.rows
78            )];
79        }
80        let mut lines = change_lines(&self.changes);
81        lines.extend(self.stale.iter().map(|line| format!("census: {line}")));
82        lines
83    }
84
85    /// Why the check fails, or `None` if it [holds](Self::holds).
86    pub fn problem(&self) -> Option<String> {
87        let worse = self
88            .changes
89            .iter()
90            .filter(|change| change.is_worse())
91            .count();
92        let mut problems = Vec::new();
93        if !self.changes.is_empty() {
94            problems.push(format!(
95                "{} row(s) differ from the accepted census, {worse} of them for the worse: a \
96                 regression beyond its slack fails here",
97                self.changes.len()
98            ));
99        }
100        if !self.stale.is_empty() {
101            problems.push(format!("{} output(s) are stale", self.stale.len()));
102        }
103        (!problems.is_empty()).then(|| problems.join("; "))
104    }
105
106    /// What to do about [`problem`](Self::problem), one line for each kind of failure; empty if
107    /// it holds. A program can write each as a `help:` line.
108    pub fn help(&self) -> Vec<String> {
109        let mut help = Vec::new();
110        if !self.changes.is_empty() {
111            help.push(format!(
112                "if the change is meant, accept it in writing with `cargo xtask census --accept \
113                 --reason <why>`, and commit {CENSUS_JSON}"
114            ));
115        }
116        if !self.stale.is_empty() {
117            help.push("run `cargo xtask census --accept`".to_owned());
118        }
119        help
120    }
121}
122
123/// A line per change, and a summary: what `cargo xtask census` prints.
124pub fn change_lines(changes: &[Change]) -> Vec<String> {
125    if changes.is_empty() {
126        return vec!["census: no row differs from the accepted census".to_owned()];
127    }
128    let mut lines: Vec<String> = changes
129        .iter()
130        .map(|change| format!("census: {}", change.describe()))
131        .collect();
132    lines.push(format!(
133        "census: {} row(s) differ from the accepted census, {} for the worse",
134        changes.len(),
135        changes.iter().filter(|change| change.is_worse()).count()
136    ));
137    lines
138}
139
140/// Why a run doesn't reproduce the committed report.
141#[derive(Debug)]
142pub enum NotReproduced {
143    /// The committed report couldn't be read.
144    Unreadable(ValidateError),
145    /// It was read, and differs from the run: the first difference.
146    Differs(String),
147}
148
149/// A run held to the committed reports.
150#[derive(Debug)]
151pub struct Checked {
152    /// The scored metrics outside their tolerance.
153    pub failed: usize,
154    /// Whether the run reproduces the committed report.
155    pub reproduced: Result<(), NotReproduced>,
156    /// The census check, or why it couldn't be taken.
157    pub census: Result<CensusCheck, ValidateError>,
158}
159
160impl Checked {
161    /// Whether every metric passed, the report reproduces, and the census holds.
162    pub fn passed(&self) -> bool {
163        self.problems().is_empty()
164    }
165
166    /// Why the check fails, in the order `cargo xtask validate --check` gives them; empty when
167    /// it passes.
168    pub fn problems(&self) -> Vec<String> {
169        let mut problems = Vec::new();
170        match &self.census {
171            Ok(census) => problems.extend(census.problem()),
172            Err(error) => problems.push(error.fact()),
173        }
174        if self.failed != 0 {
175            problems.push(format!("{} metric(s) outside tolerance", self.failed));
176        }
177        match &self.reproduced {
178            Ok(()) => {}
179            Err(NotReproduced::Unreadable(error)) => problems.push(error.fact()),
180            Err(NotReproduced::Differs(what)) => {
181                problems.push(format!("the committed report is not this run's ({what})"));
182            }
183        }
184        problems
185    }
186
187    /// What to do about the [problems](Self::problems), in the same order; empty when it passes
188    /// or nothing can be suggested. A program can write each as a `help:` line.
189    pub fn help(&self) -> Vec<String> {
190        let mut help = match &self.census {
191            Ok(census) => census.help(),
192            Err(error) => error.help().map(str::to_owned).into_iter().collect(),
193        };
194        match &self.reproduced {
195            Ok(()) => {}
196            Err(NotReproduced::Unreadable(error)) => help.extend(error.help().map(str::to_owned)),
197            Err(NotReproduced::Differs(_)) => help.push(REPORT_HINT.to_owned()),
198        }
199        help
200    }
201
202    /// What `cargo xtask validate --check` prints after the run's summary: the census's lines,
203    /// then whether the report reproduces, when every metric passed and it does.
204    pub fn lines(&self) -> Vec<String> {
205        let mut lines = match &self.census {
206            Ok(census) => census.lines(),
207            Err(_) => Vec::new(),
208        };
209        if self.failed == 0 && self.reproduced.is_ok() {
210            lines.push("validate: the committed report reproduces".to_owned());
211        }
212        lines
213    }
214}
215
216/// Holds `report`, a run of every case, to the reports committed under `root`.
217pub fn check(root: &Path, report: &Report) -> Checked {
218    let reproduced = match (read(root, LATEST_MD), read(root, LATEST_JSON)) {
219        (Ok(markdown), Ok(json)) => report
220            .reproduces(&markdown, &json)
221            .map_err(|error| NotReproduced::Differs(error.to_string())),
222        (Err(error), _) | (_, Err(error)) => Err(NotReproduced::Unreadable(error)),
223    };
224    Checked {
225        failed: report.failures().len(),
226        reproduced,
227        census: check_census(root),
228    }
229}
230
231/// The committed reports under `root` against the accepted census, and the files written from it.
232///
233/// # Errors
234///
235/// A report or the census that can't be read, a census that can't be taken, or a table file
236/// without its census block; a missing census is [`ValidateError::Case`].
237pub fn check_census(root: &Path) -> Result<CensusCheck, ValidateError> {
238    let changes = census_changes(root)?;
239    let accepted = read_accepted(root)?.ok_or_else(|| {
240        ValidateError::Case(format!("{CENSUS_JSON} is missing; {CENSUS_MISSING_HINT}"))
241    })?;
242    let stale = stale(census_outputs(root, &accepted)?, |path| {
243        std::fs::read_to_string(root.join(path))
244    });
245    Ok(CensusCheck {
246        rows: accepted.census.rows.len(),
247        changes,
248        stale,
249    })
250}
251
252/// The outputs whose committed text, as `read` gives it, isn't what the census writes.
253pub fn stale(
254    outputs: Vec<(PathBuf, String)>,
255    read: impl Fn(&Path) -> std::io::Result<String>,
256) -> Vec<String> {
257    outputs
258        .into_iter()
259        .filter_map(|(path, text)| match read(&path) {
260            Ok(committed) if committed == text => None,
261            Ok(_) => Some(format!(
262                "{} is not what the accepted census writes",
263                path.display()
264            )),
265            Err(error) => Some(format!("{}: {error}", path.display())),
266        })
267        .collect()
268}
269
270/// The differences between the committed reports and the accepted census; every row is added
271/// when none is accepted yet.
272///
273/// # Errors
274///
275/// As [`take_census`] and [`read_accepted`].
276pub fn census_changes(root: &Path) -> Result<Vec<Change>, ValidateError> {
277    let now = take_census(root)?;
278    let accepted = read_accepted(root)?;
279    let empty = Census {
280        slack_share: census::SLACK_SHARE,
281        references: now.references.clone(),
282        rows: Vec::new(),
283    };
284    Ok(census::compare(
285        accepted
286            .as_ref()
287            .map_or(&empty, |accepted| &accepted.census),
288        &now,
289    ))
290}
291
292/// Every file the accepted census writes, with its text, by its path from `root`.
293///
294/// Everything is rendered from the accepted rows, its summaries computed again, so a summary
295/// edited by hand, or one an older version of the code wrote, shows as a stale `census.json`.
296///
297/// # Errors
298///
299/// A table file that can't be read or has no single census block.
300pub fn census_outputs(
301    root: &Path,
302    accepted: &Accepted,
303) -> Result<Vec<(PathBuf, String)>, ValidateError> {
304    let accepted = accepted.refreshed();
305    let json = serde_json::to_string_pretty(&accepted).map_err(|source| ValidateError::Json {
306        path: CENSUS_JSON.to_owned(),
307        source: Box::new(source),
308    })?;
309    let mut files = vec![
310        (PathBuf::from(CENSUS_JSON), format!("{json}\n")),
311        (PathBuf::from(CENSUS_MD), accepted.to_markdown()),
312    ];
313    for (name, svg) in census::badges(&accepted.summaries) {
314        files.push((Path::new(BADGES).join(name), svg));
315    }
316    let block = format!(
317        "{BEGIN}\n\n{}\n{END}",
318        census::table(&accepted.summaries, Some(CENSUS_URL))
319    );
320    for path in TABLES {
321        let text = read(root, path)?;
322        files.push((
323            PathBuf::from(path),
324            splice(&text, &block, path).map_err(ValidateError::Case)?,
325        ));
326    }
327    Ok(files)
328}
329
330/// `text` with what lies from [`BEGIN`] to [`END`] replaced by `block`.
331///
332/// # Errors
333///
334/// No block, one without its end, or two blocks, each naming `path`.
335pub fn splice(text: &str, block: &str, path: &str) -> Result<String, String> {
336    let start = text
337        .find(BEGIN)
338        .ok_or_else(|| format!("{path} has no census block: add the line `{BEGIN}` and `{END}`"))?;
339    let end = text[start..]
340        .find(END)
341        .map(|at| start + at + END.len())
342        .ok_or_else(|| format!("{path}: the census block has no `{END}`"))?;
343    if text[end..].contains(BEGIN) {
344        return Err(format!("{path} has two census blocks"));
345    }
346    Ok(format!("{}{block}{}", &text[..start], &text[end..]))
347}
348
349/// The accepted census, or `None` if none is committed.
350///
351/// # Errors
352///
353/// A census file that can't be read or isn't one.
354pub fn read_accepted(root: &Path) -> Result<Option<Accepted>, ValidateError> {
355    if root.join(CENSUS_JSON).exists() {
356        read_json(root, CENSUS_JSON).map(Some)
357    } else {
358        Ok(None)
359    }
360}
361
362/// The census of the reports committed under `root`.
363///
364/// # Errors
365///
366/// A report that can't be read or parsed, or a census that can't be taken from them.
367pub fn take_census(root: &Path) -> Result<Census, ValidateError> {
368    let harness: Report = read_json(root, LATEST_JSON)?;
369    let real: RealFlightReport = read_json(root, "validation/reports/real-flights.json")?;
370    let examples: Value = read_json(root, "validation/reports/openrocket-flights.json")?;
371    let library: Value = read_json(root, "validation/reports/openrocket-library-flights.json")?;
372    Census::take(Reports {
373        harness: &harness,
374        real_flights: &real,
375        openrocket_examples: &examples,
376        openrocket_library: &library,
377    })
378    .map_err(|error| ValidateError::Case(error.to_string()))
379}
380
381fn read(root: &Path, relative: &str) -> Result<String, ValidateError> {
382    let path = root.join(relative);
383    std::fs::read_to_string(&path).map_err(|source| ValidateError::Io {
384        what: "reading",
385        path: path.display().to_string(),
386        source,
387    })
388}
389
390fn read_json<T: serde::de::DeserializeOwned>(
391    root: &Path,
392    relative: &str,
393) -> Result<T, ValidateError> {
394    serde_json::from_str(&read(root, relative)?).map_err(|source| ValidateError::Json {
395        path: root.join(relative).display().to_string(),
396        source: Box::new(source),
397    })
398}
399
400#[cfg(test)]
401mod tests {
402    use super::*;
403
404    /// Copies `from` into `to`, folders and all.
405    fn copy_tree(from: &Path, to: &Path) {
406        std::fs::create_dir_all(to).unwrap();
407        for entry in std::fs::read_dir(from).unwrap() {
408            let entry = entry.unwrap();
409            let target = to.join(entry.file_name());
410            if entry.file_type().unwrap().is_dir() {
411                copy_tree(&entry.path(), &target);
412            } else {
413                std::fs::copy(entry.path(), &target).unwrap();
414            }
415        }
416    }
417
418    /// Replaces the first `from` in the file at `path` with `to`, and returns what it held.
419    fn spoil(path: &Path, from: &str, to: &str) -> String {
420        let text = std::fs::read_to_string(path).unwrap();
421        assert!(text.contains(from), "{} has no `{from}`", path.display());
422        std::fs::write(path, text.replacen(from, to, 1)).unwrap();
423        text
424    }
425
426    /// The check, end to end, on a copy of what it reads (the validation folder and the files
427    /// written from the census): the copy passes, then fails spoiled each way the check looks at,
428    /// with its reasons. `cargo xtask validate --check` calls the same `check` (ADR-107); the
429    /// test moved here from `hpr validate`'s when that command left the command line (ADR-187).
430    #[test]
431    fn a_copy_passes_and_fails_where_it_is_spoiled() {
432        let root = crate::tests::root();
433        let dir = tempfile::tempdir().unwrap();
434        let at = dir.path();
435        copy_tree(&root.join("validation"), &at.join("validation"));
436        copy_tree(&root.join("docs/images"), &at.join("docs/images"));
437        for file in ["README.md", "docs/accuracy.md"] {
438            std::fs::copy(root.join(file), at.join(file)).unwrap();
439        }
440        let report = crate::run_lock(at, false).unwrap();
441        let checked = check(at, &report);
442        assert!(
443            checked.passed(),
444            "the copy is whole: {:?}",
445            checked.problems()
446        );
447        assert_eq!(
448            checked.lines().last().map(String::as_str),
449            Some("validate: the committed report reproduces")
450        );
451
452        // A committed report that isn't this run's.
453        let latest = at.join(LATEST_JSON);
454        let committed = std::fs::read_to_string(&latest).unwrap();
455        let mut json: serde_json::Value = serde_json::from_str(&committed).unwrap();
456        let measured = &mut json["comparisons"][0]["measured"];
457        *measured = serde_json::Value::from(measured.as_f64().unwrap() * 1.01);
458        std::fs::write(&latest, serde_json::to_string_pretty(&json).unwrap()).unwrap();
459        let checked = check(at, &report);
460        assert!(checked.reproduced.is_err());
461        let problems = checked.problems();
462        assert_eq!(problems.len(), 1, "{problems:?}");
463        assert!(
464            problems[0].starts_with("the committed report is not this run's"),
465            "{problems:?}"
466        );
467        assert_eq!(checked.help(), [REPORT_HINT]);
468        std::fs::write(&latest, committed).unwrap();
469
470        // A file written from the census, edited by hand.
471        let readme = at.join("README.md");
472        let was = spoil(
473            &readme,
474            "<!-- census: end -->",
475            "edited\n<!-- census: end -->",
476        );
477        let checked = check(at, &report);
478        assert!(checked.reproduced.is_ok());
479        assert_eq!(
480            checked.census.as_ref().unwrap().stale,
481            ["README.md is not what the accepted census writes"]
482        );
483        assert_eq!(checked.problems(), ["1 output(s) are stale"]);
484        assert_eq!(checked.help(), ["run `cargo xtask census --accept`"]);
485        std::fs::write(&readme, was).unwrap();
486        assert!(check(at, &report).passed(), "restored");
487
488        // A metric outside its tolerance, which also moves the report.
489        let case = at.join("validation/cases/descent-valetudo.toml");
490        spoil(&case, "relative = 0.03", "relative = 1e-12");
491        let report = crate::run_lock(at, false).unwrap();
492        let checked = check(at, &report);
493        assert_eq!(checked.failed, 1);
494        assert_eq!(checked.failed, report.failures().len());
495        let problems = checked.problems();
496        assert!(
497            problems.contains(&"1 metric(s) outside tolerance".to_owned()),
498            "{problems:?}"
499        );
500        assert!(!checked.passed());
501        assert!(
502            !checked
503                .lines()
504                .contains(&"validate: the committed report reproduces".to_owned())
505        );
506    }
507
508    /// A census check of 7 rows, none changed, with `stale` stale files.
509    fn census(stale: usize) -> CensusCheck {
510        CensusCheck {
511            rows: 7,
512            changes: Vec::new(),
513            stale: vec!["README.md is not what the accepted census writes".to_owned(); stale],
514        }
515    }
516
517    /// The reasons, in xtask's order, and the lines after the summary.
518    #[test]
519    fn a_check_says_each_reason_once_in_order() {
520        let passed = Checked {
521            failed: 0,
522            reproduced: Ok(()),
523            census: Ok(census(0)),
524        };
525        assert!(passed.passed());
526        assert_eq!(
527            passed.lines(),
528            [
529                "census: the committed reports hold to the accepted census (7 rows)",
530                "validate: the committed report reproduces"
531            ]
532        );
533        let failed = Checked {
534            failed: 2,
535            reproduced: Err(NotReproduced::Differs("latest.json: x".to_owned())),
536            census: Ok(census(1)),
537        };
538        assert_eq!(
539            failed.problems(),
540            [
541                "1 output(s) are stale",
542                "2 metric(s) outside tolerance",
543                "the committed report is not this run's (latest.json: x)"
544            ]
545        );
546        assert_eq!(
547            failed.help(),
548            [
549                "run `cargo xtask census --accept`",
550                "run `cargo xtask validate` and commit validation/reports/latest.{md,json}"
551            ]
552        );
553        assert_eq!(
554            failed.lines(),
555            [
556                "census: no row differs from the accepted census",
557                "census: README.md is not what the accepted census writes"
558            ]
559        );
560        // An unreadable report is its own reason, not a report to regenerate; a census that can't
561        // be taken prints no census line.
562        let unread = Checked {
563            failed: 0,
564            reproduced: Err(NotReproduced::Unreadable(ValidateError::Case(
565                "reading x".into(),
566            ))),
567            census: Err(ValidateError::Case("no census".into())),
568        };
569        assert_eq!(unread.problems(), ["no census", "reading x"]);
570        assert!(unread.help().is_empty());
571        // A missing census says what to run, apart from the fact.
572        let missing = Checked {
573            failed: 0,
574            reproduced: Ok(()),
575            census: Err(ValidateError::Case(format!(
576                "{CENSUS_JSON} is missing; {CENSUS_MISSING_HINT}"
577            ))),
578        };
579        assert_eq!(
580            missing.problems(),
581            ["validation/reports/census.json is missing"]
582        );
583        assert_eq!(missing.help(), [CENSUS_MISSING_HINT]);
584        assert!(unread.lines().is_empty());
585    }
586
587    #[test]
588    fn a_block_is_replaced_whole_and_only_once() {
589        let text = format!("before\n{BEGIN}\nold\n{END}\nafter\n");
590        let block = format!("{BEGIN}\nnew\n{END}");
591        assert_eq!(
592            splice(&text, &block, "f").unwrap(),
593            format!("before\n{BEGIN}\nnew\n{END}\nafter\n")
594        );
595        assert!(
596            splice("no block", &block, "f")
597                .unwrap_err()
598                .contains("no census block")
599        );
600        assert!(
601            splice(&format!("{BEGIN}\nold"), &block, "f")
602                .unwrap_err()
603                .contains("no `")
604        );
605        let twice = format!("{text}{text}");
606        assert!(splice(&twice, &block, "f").unwrap_err().contains("two"));
607    }
608}
hpr_validate/committed.rs

hpr_validate/
committed.rs
