hpr_validate/
summary.rs

1//! A run's report in a line per case and one for the whole: what `cargo xtask validate` prints.
2//!
3//! The worst figure is over the **scored** metrics: a metric a case declares not scored is
4//! counted and named separately, so a run cannot look green by leaving something out and cannot
5//! look alarming because a declared difference is large in percentage terms.
6
7use std::fmt;
8
9use serde::{Deserialize, Serialize};
10
11use crate::report::{Comparison, Report, Verdict};
12
13/// How one case came out.
14#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
15#[serde(tag = "kind", rename_all = "snake_case")]
16pub enum CaseSummary {
17    /// A known gap: hpr refuses the flight, so nothing is compared.
18    Gap {
19        /// The case's id.
20        case: String,
21        /// The metrics the case would compare.
22        metrics: usize,
23        /// Why hpr refuses it.
24        refusal: String,
25    },
26    /// Predicted mode: every metric reported against a target, none gating.
27    Predicted {
28        /// The case's id.
29        case: String,
30        /// The metrics reported.
31        metrics: usize,
32        /// How many are within their target.
33        within_target: usize,
34        /// The metric furthest from its reference, and its relative difference.
35        largest: Option<(String, f64)>,
36    },
37    /// Every metric held to its tolerance.
38    Scored {
39        /// The case's id.
40        case: String,
41        /// The metrics compared.
42        metrics: usize,
43        /// The largest relative difference among the scored metrics, as a magnitude.
44        worst_scored: f64,
45        /// The metrics the case declares not scored.
46        not_scored: Vec<String>,
47        /// How many metrics are outside their tolerance.
48        failed: usize,
49    },
50}
51
52impl CaseSummary {
53    /// The case's id.
54    pub fn case(&self) -> &str {
55        match self {
56            Self::Gap { case, .. } | Self::Predicted { case, .. } | Self::Scored { case, .. } => {
57                case
58            }
59        }
60    }
61}
62
63impl fmt::Display for CaseSummary {
64    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
65        match self {
66            // A gap compares nothing, so "0 metrics, worst +0.00%" would read as a clean pass.
67            Self::Gap {
68                case,
69                metrics,
70                refusal,
71            } => write!(
72                f,
73                "{case}: known gap, {metrics} metric(s) not scored: hpr {refusal}"
74            ),
75            // Predicted mode reports against a target and never gates, so it has no "worst
76            // scored"; its largest difference is the headline.
77            Self::Predicted {
78                case,
79                metrics,
80                within_target,
81                largest,
82            } => {
83                write!(
84                    f,
85                    "{case}: predicted, {metrics} metric(s) reported, {within_target} within target"
86                )?;
87                if let Some((metric, relative)) = largest {
88                    write!(f, ", largest {metric} {:+.2}%", 100.0 * relative)?;
89                }
90                Ok(())
91            }
92            Self::Scored {
93                case,
94                metrics,
95                worst_scored,
96                not_scored,
97                failed,
98            } => {
99                write!(
100                    f,
101                    "{case}: {metrics} metric(s), worst scored {:+.2}%",
102                    100.0 * worst_scored
103                )?;
104                if !not_scored.is_empty() {
105                    write!(f, ", not scored: {}", not_scored.join(", "))?;
106                }
107                if *failed != 0 {
108                    write!(f, ", {failed} OUT OF TOLERANCE")?;
109                }
110                Ok(())
111            }
112        }
113    }
114}
115
116/// The whole run, counted.
117#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
118pub struct Totals {
119    /// The cases run.
120    pub cases: usize,
121    /// The metrics compared or reported.
122    pub metrics: usize,
123    /// The metrics their cases declare not scored.
124    pub not_scored: usize,
125    /// The metrics reported against a target (predicted mode).
126    pub predicted: usize,
127    /// How many of those are outside their target.
128    pub outside_target: usize,
129    /// The scored metrics outside their tolerance.
130    pub failed: usize,
131}
132
133impl Totals {
134    /// Whether every scored metric is within its tolerance.
135    pub fn passed(&self) -> bool {
136        self.failed == 0
137    }
138}
139
140impl fmt::Display for Totals {
141    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142        let predicted = format!(
143            "predicted, against a target, {} outside it",
144            self.outside_target
145        );
146        let aside: Vec<String> = [
147            (self.not_scored, "not scored"),
148            (self.predicted, predicted.as_str()),
149        ]
150        .into_iter()
151        .filter(|(count, _)| *count > 0)
152        .map(|(count, what)| format!("{count} {what}"))
153        .collect();
154        write!(
155            f,
156            "validate: {} case(s), {} metric(s){}, {}",
157            self.cases,
158            self.metrics,
159            if aside.is_empty() {
160                String::new()
161            } else {
162                format!(" ({})", aside.join(", "))
163            },
164            if self.passed() { "ok" } else { "FAILED" }
165        )
166    }
167}
168
169/// Each case of `report`, in its order.
170pub fn cases(report: &Report) -> Vec<CaseSummary> {
171    report
172        .cases
173        .iter()
174        .map(|case| {
175            if let Some(gap) = report.gaps.iter().find(|gap| gap.case == *case) {
176                return CaseSummary::Gap {
177                    case: case.clone(),
178                    metrics: gap.metric_count,
179                    refusal: gap.refusal.clone(),
180                };
181            }
182            let metrics: Vec<&Comparison> = report
183                .comparisons
184                .iter()
185                .filter(|comparison| comparison.case == *case)
186                .collect();
187            if metrics.iter().any(|comparison| comparison.targeted_row()) {
188                return CaseSummary::Predicted {
189                    case: case.clone(),
190                    metrics: metrics.len(),
191                    within_target: metrics
192                        .iter()
193                        .filter(|comparison| comparison.verdict == Verdict::WithinTarget)
194                        .count(),
195                    largest: metrics
196                        .iter()
197                        .filter_map(|comparison| {
198                            comparison.relative.map(|r| (comparison.metric.clone(), r))
199                        })
200                        .max_by(|(_, a), (_, b)| a.abs().total_cmp(&b.abs())),
201                };
202            }
203            CaseSummary::Scored {
204                case: case.clone(),
205                metrics: metrics.len(),
206                worst_scored: metrics
207                    .iter()
208                    .filter(|comparison| comparison.scored())
209                    .filter_map(|comparison| comparison.relative)
210                    .fold(0.0_f64, |worst, relative| worst.max(relative.abs())),
211                not_scored: metrics
212                    .iter()
213                    .filter(|comparison| !comparison.scored())
214                    .map(|comparison| comparison.metric.clone())
215                    .collect(),
216                failed: metrics
217                    .iter()
218                    .filter(|comparison| comparison.verdict == Verdict::Fail)
219                    .count(),
220            }
221        })
222        .collect()
223}
224
225/// The whole of `report`, counted.
226pub fn totals(report: &Report) -> Totals {
227    let count = |verdict| {
228        report
229            .comparisons
230            .iter()
231            .filter(|comparison| comparison.verdict == verdict)
232            .count()
233    };
234    Totals {
235        cases: report.cases.len(),
236        metrics: report.comparisons.len(),
237        not_scored: report.not_scored().len(),
238        predicted: report
239            .comparisons
240            .iter()
241            .filter(|comparison| comparison.targeted_row())
242            .count(),
243        outside_target: count(Verdict::OutsideTarget),
244        failed: report.failures().len(),
245    }
246}
hpr_validate/summary.rs

hpr_validate/
summary.rs
