Skip to main content

hpr_flightdata/
custom.rs

1//! A flyer's own logger, read by a column mapping or a record layout stated in a JSON file: what
2//! `hpr analyze` and the Python package take ([ADR-276][adr-276];
3//! [M7.1d2, your own logger from the command line and Python][m7-1d2]).
4//!
5//! [`generic::read_with`], [`records::read`] and [`capture::read`] read a log by a mapping or a
6//! layout a program builds. This module reads the mapping or layout from the JSON file a person
7//! writes, and the log's bytes by it, so that the command line, Python and a Rust program read
8//! one file into one record:
9//!
10//! - **A mapping file** is a [`Mapping`] as JSON: how a table's columns are read, and which one is
11//!   the primary track, if one is ([`Mapping::primary`]). [`header_mapping`] gives the one a
12//!   table's header states, to edit. Its schema is
13//!   [`schema/cli/analyze-mapping.schema.json`][mapping-schema].
14//! - **A layout file** is a [`Layout`]: each field of a record, its name stating its unit as a
15//!   table's header does, the byte order, and for a serial console's capture, the block the
16//!   records are in. Its schema is [`schema/cli/analyze-layout.schema.json`][layout-schema].
17//! - **A FIP header file** is a [`FipHeader`]: the header line a Featherweight Raven's FIP export
18//!   lost when it was pasted into a workbook below none, and the columns a table pasted over it
19//!   covers ([ADR-274][adr-274]), read by [`read_by_fip_header`]. Its schema is
20//!   [`schema/cli/analyze-fip-header.schema.json`][fip-header-schema].
21//!
22//! **Refusals.** A file is refused naming the field it fails by its path in the JSON, such as
23//! `columns[2].unit`, with its line and column, and a key the schema doesn't name is refused
24//! too, so that a misspelled one isn't silently left out. A file is at most [`MAX_FILE_BYTES`].
25//! What the JSON can't check, such as a unit that doesn't measure its column's quantity, the
26//! reader refuses when it reads the log, naming the column.
27//!
28//! **A workbook.** A mapping reads a table from an `.xlsx` workbook's sheet, its cells as the
29//! workbook stores them ([`workbook::sheets`]), when the caller names the sheet, from 0. A
30//! workbook read with no sheet named is refused, its sheets listed, rather than one picked.
31//!
32//! [adr-274]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/docs/decisions/0274-the-rest-of-the-importers-planned-teensy-logger-first.md
33//! [adr-276]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/docs/decisions/0276-the-2026-10-16-own-loggers-and-gpx-in-the-analyzer.md
34//! [m7-1d2]: https://hpr.fusionspace.co/decisions-and-roadmap.html#m7-1d2
35//! [mapping-schema]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/schema/cli/analyze-mapping.schema.json
36//! [layout-schema]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/schema/cli/analyze-layout.schema.json
37//! [fip-header-schema]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/schema/cli/analyze-fip-header.schema.json
38
39use std::borrow::Cow;
40
41use schemars::JsonSchema;
42use serde::de::DeserializeOwned;
43use serde::{Deserialize, Serialize};
44
45use crate::capture;
46use crate::error::LogError;
47use crate::generic::{self, Mapping};
48use crate::import;
49use crate::log::FlightLog;
50use crate::records::{self, ByteOrder, Field, RecordLayout};
51use crate::{archive, raven, workbook, xls};
52
53/// The most bytes a mapping or a layout file may hold, 1 MiB: [`generic::MAX_COLUMNS`] columns
54/// or fields, each with a name of hundreds of characters, take a few tens of kilobytes.
55pub const MAX_FILE_BYTES: usize = 1 << 20;
56
57/// A record layout as its file states it: the fields of a record, their byte order, and for a
58/// serial console's capture, the block the records are in.
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
60#[serde(deny_unknown_fields)]
61pub struct Layout {
62    /// Each field of a record, in the record's order. A field's name states its unit as a
63    /// table's header does, such as `Time (ms)` or `pressure__pa`: the records are written out
64    /// as a table under those names and read by what they state ([`records::read`]), so one
65    /// field must be a clock in a unit of time.
66    pub fields: Vec<Field>,
67    /// The order of each field's bytes: little-endian unless the layout says otherwise.
68    #[serde(default)]
69    pub byte_order: ByteOrder,
70    /// For a serial console's capture of the records as base64, the name of the block they are
71    /// in, as the capture's start marker names it ([`capture`]); `None` for a file of the
72    /// records' bytes.
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub block: Option<String>,
75}
76
77impl Layout {
78    /// The fields and byte order, as [`records::read`] and [`capture::read`] take them.
79    pub fn record_layout(&self) -> RecordLayout {
80        RecordLayout {
81            fields: self.fields.clone(),
82            byte_order: self.byte_order,
83        }
84    }
85}
86
87/// Reads a mapping file's JSON.
88///
89/// # Errors
90///
91/// [`LogError::Mapping`] for text of more than [`MAX_FILE_BYTES`], or JSON that isn't a mapping:
92/// a key missing or not the schema's, or a value of the wrong kind, naming its path and line.
93pub fn mapping_from_json(text: &str) -> Result<Mapping, LogError> {
94    from_json(text, "mapping").map_err(|message| LogError::Mapping { message })
95}
96
97/// Reads a layout file's JSON.
98///
99/// # Errors
100///
101/// [`LogError::Layout`] as [`mapping_from_json`] says for a mapping.
102pub fn layout_from_json(text: &str) -> Result<Layout, LogError> {
103    from_json(text, "layout").map_err(|message| LogError::Layout { message })
104}
105
106/// A header stated for a Featherweight Raven's FIP export that lost its own when it was pasted
107/// into a workbook below none ([ADR-274][adr-274]): the export's header line, and the columns a
108/// table pasted over the export covers.
109///
110/// [adr-274]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/docs/decisions/0274-the-rest-of-the-importers-planned-teensy-logger-first.md
111#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
112#[serde(deny_unknown_fields)]
113pub struct FipHeader {
114    /// The export's header line, as a FIP export of the same channels writes it: each channel's
115    /// `Time@` clock, its value and its `bILBA` status, such as
116    /// `Time@Baro (Atm),Baro (Atm),bILBA,...`.
117    pub header: String,
118    /// The columns, from 1, of a table pasted over the export, whose cells hide whether a
119    /// channel has a sample in their rows; none when left out.
120    #[serde(default)]
121    #[schemars(inner(range(min = 1)))]
122    pub pasted: Vec<usize>,
123}
124
125/// Reads a FIP header file's JSON.
126///
127/// # Errors
128///
129/// [`LogError::FipHeader`] as [`mapping_from_json`] says for a mapping.
130pub fn fip_header_from_json(text: &str) -> Result<FipHeader, LogError> {
131    from_json(text, "FIP header").map_err(|message| LogError::FipHeader { message })
132}
133
134/// Reads a Raven FIP export under the header stated for it ([`raven::read_rows_around`]): the
135/// file as text, or the sheet `sheet` of a spreadsheet's workbook, from 0, its cells as the
136/// workbook stores them; with no sheet chosen, a workbook's one sheet.
137///
138/// # Errors
139///
140/// [`LogError::Choice`] for a sheet chosen of a file that isn't a workbook, a sheet the workbook
141/// doesn't hold, or none chosen of a workbook of several; a workbook's refusal; otherwise as
142/// [`raven::read_rows_around`].
143pub fn read_by_fip_header(
144    bytes: &[u8],
145    stated: &FipHeader,
146    sheet: Option<usize>,
147) -> Result<FlightLog, LogError> {
148    let text = match (workbook_sheets(bytes).transpose()?, sheet) {
149        (None, None) => import::decode(bytes).into_owned(),
150        (None, Some(_)) => {
151            return Err(LogError::Choice {
152                message: "the file isn't a spreadsheet's workbook, so it has no sheet to choose"
153                    .to_owned(),
154            });
155        }
156        (Some(sheets), chosen) => {
157            let count = sheets.len();
158            let index = match chosen {
159                Some(index) => index,
160                None if count == 1 => 0,
161                None => {
162                    return Err(LogError::Choice {
163                        message: format!("the workbook has {count} sheets: choose one, from 0"),
164                    });
165                }
166            };
167            sheets
168                .into_iter()
169                .nth(index)
170                .ok_or(LogError::Choice {
171                    message: format!(
172                        "the workbook has {count} {}, numbered from 0: it has no sheet {index}",
173                        if count == 1 { "sheet" } else { "sheets" }
174                    ),
175                })?
176                .text
177        }
178    };
179    // The rows' reader words its refusals of a stated header as a mapping's; here they are the
180    // FIP header's.
181    raven::read_rows_around(&stated.header, &text, &stated.pasted).map_err(|error| match error {
182        LogError::Mapping { message } => LogError::FipHeader { message },
183        other => other,
184    })
185}
186
187/// A mapping's or a layout's file as text: UTF-8, a byte-order mark dropped, or UTF-16 with its
188/// byte-order mark, as Windows PowerShell 5.1 writes what `>` sends to a file; `None` for any
189/// other bytes, among them UTF-16 with no mark, whose zero bytes no JSON text holds.
190#[must_use]
191pub fn json_text(bytes: &[u8]) -> Option<String> {
192    let utf16 = |rest: &[u8], unit: fn([u8; 2]) -> u16| {
193        let (pairs, odd) = rest.as_chunks::<2>();
194        if !odd.is_empty() {
195            return None;
196        }
197        char::decode_utf16(pairs.iter().map(|pair| unit(*pair)))
198            .collect::<Result<String, _>>()
199            .ok()
200    };
201    match bytes {
202        [0xEF, 0xBB, 0xBF, rest @ ..] => std::str::from_utf8(rest).ok().map(str::to_owned),
203        [0xFF, 0xFE, rest @ ..] => utf16(rest, u16::from_le_bytes),
204        [0xFE, 0xFF, rest @ ..] => utf16(rest, u16::from_be_bytes),
205        // JSON escapes a NUL inside a string, so a zero byte isn't UTF-8 JSON: most likely
206        // UTF-16 with no mark.
207        _ if bytes.contains(&0) => None,
208        _ => std::str::from_utf8(bytes).ok().map(str::to_owned),
209    }
210}
211
212/// The JSON read as a `T`, or why not: the path of the field it fails, then serde's words, which
213/// end in the line and column.
214fn from_json<T: DeserializeOwned>(text: &str, what: &str) -> Result<T, String> {
215    if text.len() > MAX_FILE_BYTES {
216        return Err(format!(
217            "a {what} file is at most {MAX_FILE_BYTES} bytes; this one is {}",
218            text.len()
219        ));
220    }
221    let mut json = serde_json::Deserializer::from_str(text);
222    let value = serde_path_to_error::deserialize(&mut json).map_err(|error| {
223        let words = error.inner().to_string();
224        let mut segments: Vec<_> = error.path().iter().collect();
225        // The path of a key the schema doesn't name ends at the key, which serde's words name.
226        if words.starts_with("unknown field") {
227            segments.pop();
228        }
229        // serde's words quote the value refused, which can be as long as the file.
230        let words = cut(&words, MAX_WORDS_CHARS);
231        match path(&segments) {
232            path if path.is_empty() => words,
233            path => format!("`{path}`: {words}"),
234        }
235    })?;
236    json.end().map_err(|error| error.to_string())?;
237    Ok(value)
238}
239
240/// The most characters of serde's words a refusal quotes: room for the value refused and the
241/// list of every value a field takes (a quantity's 20).
242const MAX_WORDS_CHARS: usize = 1000;
243
244/// Text cut to a number of characters, its last a `…` where it is cut.
245fn cut(text: &str, chars: usize) -> String {
246    if text.chars().count() <= chars {
247        text.to_owned()
248    } else {
249        let mut cut: String = text.chars().take(chars - 1).collect();
250        cut.push('…');
251        cut
252    }
253}
254
255/// A field's path in the JSON, as `columns[2].unit`; empty for the document itself.
256fn path(segments: &[&serde_path_to_error::Segment]) -> String {
257    use serde_path_to_error::Segment;
258    let mut text = String::new();
259    for segment in segments {
260        let key = match segment {
261            Segment::Seq { index } => {
262                text.push_str(&format!("[{index}]"));
263                continue;
264            }
265            Segment::Map { key } => key.as_str(),
266            Segment::Enum { variant } => variant.as_str(),
267            Segment::Unknown => "?",
268        };
269        if !text.is_empty() {
270            text.push('.');
271        }
272        text.push_str(key);
273    }
274    text
275}
276
277/// What a table's header states, as a mapping to edit, with a note on each column it leaves
278/// kept as written, saying why ([`generic::header_reading`]). `sheet` names a spreadsheet's
279/// workbook's sheet, `.xlsx` or `.xls`, from 0; `None` for a text file, or a workbook of one
280/// sheet.
281///
282/// # Errors
283///
284/// [`LogError::NotThisFormat`] for a table whose header doesn't name a clock with a unit of time;
285/// [`LogError::Mapping`] for a workbook of several sheets with none named, or a sheet it doesn't
286/// have, or a sheet named for a file that isn't a workbook.
287pub fn header_mapping(
288    bytes: &[u8],
289    sheet: Option<usize>,
290) -> Result<(Mapping, Vec<String>), LogError> {
291    generic::header_reading(&table_text(bytes, sheet)?)
292}
293
294/// Reads a table by a mapping, as [`generic::read_with`] does: from a text file, or from the
295/// sheet `sheet`, from 0, of a spreadsheet's workbook, `.xlsx` or `.xls`, or its one sheet.
296///
297/// # Errors
298///
299/// As [`header_mapping`] for the sheet; otherwise as [`generic::read_with`].
300pub fn read_by_mapping(
301    bytes: &[u8],
302    mapping: &Mapping,
303    sheet: Option<usize>,
304) -> Result<FlightLog, LogError> {
305    generic::read_with(&table_text(bytes, sheet)?, mapping)
306}
307
308/// Reads binary records by a layout ([`records::read`]), or a serial console's capture of them,
309/// the block the layout names ([`capture::read`]).
310///
311/// # Errors
312///
313/// [`LogError::Layout`] for a capture read by a layout that names no block, or a block named for
314/// a file that isn't a capture; otherwise as [`records::read`] or [`capture::read`].
315pub fn read_by_layout(bytes: &[u8], layout: &Layout) -> Result<FlightLog, LogError> {
316    let text = import::decode(bytes);
317    let is_capture = capture::recognizes(&text);
318    match (&layout.block, is_capture) {
319        (Some(block), true) => capture::read(&text, block, &layout.record_layout()),
320        (Some(block), false) => Err(LogError::Layout {
321            message: format!(
322                "the layout names the block {block:?} of a serial console's capture, and the \
323                 file isn't one. For a file of the records' bytes, leave `block` out"
324            ),
325        }),
326        (None, true) => {
327            let names: Vec<String> = capture::blocks(&text)?
328                .into_iter()
329                .map(|block| format!("{:?}", block.name))
330                .collect();
331            Err(LogError::Layout {
332                message: format!(
333                    "the file is a serial console's capture of records as base64, its blocks {}. \
334                     The layout must name the block its records are in, as `block`",
335                    names.join(", ")
336                ),
337            })
338        }
339        (None, false) => records::read(bytes, &layout.record_layout()),
340    }
341}
342
343/// A spreadsheet's workbook's sheets, `.xlsx` or `.xls`; `None` for a file that is neither.
344fn workbook_sheets(bytes: &[u8]) -> Option<Result<Vec<workbook::Sheet>, LogError>> {
345    if archive::is_archive(bytes) {
346        Some(workbook::sheets(bytes))
347    } else if xls::recognizes(bytes) {
348        Some(xls::sheets(bytes))
349    } else {
350        None
351    }
352}
353
354/// The table's text: the file's, or a sheet's of a spreadsheet's workbook, `.xlsx` or `.xls`:
355/// the sheet named, or the workbook's one sheet.
356fn table_text(bytes: &[u8], sheet: Option<usize>) -> Result<Cow<'_, str>, LogError> {
357    let refused = |message: String| LogError::Mapping { message };
358    match (workbook_sheets(bytes), sheet) {
359        (Some(Ok(sheets)), Some(sheet)) => {
360            let count = sheets.len();
361            sheets
362                .into_iter()
363                .nth(sheet)
364                .map(|found| Cow::Owned(found.text))
365                .ok_or_else(|| {
366                    refused(format!(
367                        "the workbook has no sheet {sheet}: {}",
368                        match count {
369                            0 => "it has none".to_owned(),
370                            1 => "its one sheet is 0".to_owned(),
371                            n => format!("its {n} sheets are 0 to {}", n - 1),
372                        }
373                    ))
374                })
375        }
376        (Some(Err(error)), Some(_)) => Err(refused(format!(
377            "a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook that reads: {error}"
378        ))),
379        (None, Some(_)) => Err(refused(
380            "a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook".to_owned(),
381        )),
382        (Some(Ok(mut sheets)), None) if sheets.len() == 1 => {
383            Ok(Cow::Owned(sheets.swap_remove(0).text))
384        }
385        (Some(Ok(sheets)), None) => {
386            let names = workbook::listed(
387                sheets.iter().map(|found| found.name.as_str()).enumerate(),
388                |index, name| format!("{index} {name:?}"),
389            );
390            Err(refused(format!(
391                "the file is a spreadsheet's workbook, its sheets {}: name the sheet the table is \
392                 on, from 0",
393                if names.is_empty() { "none" } else { &names }
394            )))
395        }
396        // An archive that isn't a workbook, read as the text it holds, as before workbooks were;
397        // a workbook that fails is refused for its own fault.
398        (Some(Err(LogError::NotThisFormat { .. })) | None, None) => Ok(import::decode(bytes)),
399        (Some(Err(error)), None) => Err(error),
400    }
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406    use crate::log::Quantity;
407
408    /// An invented logger's table: a clock in milliseconds, a barometric altitude above the pad
409    /// in feet, an acceleration in g.
410    const TABLE: &str = "\
411t (ms),alt (ft),accel (g)
4120,0,1.0
41350,1.5,6.2
414100,6.0,6.1
415150,13.4,0.4
416";
417
418    fn mapping(primary: &str) -> String {
419        format!(
420            r#"{{
421  "header_line": 1,
422  "separator": ",",
423  "decimal_comma": false,
424  "clock": 0,
425  "primary": {primary},
426  "columns": [
427    {{"name": "t (ms)", "quantity": null, "unit": "ms"}},
428    {{"name": "alt (ft)", "quantity": "altitude_above_ground", "unit": "ft"}},
429    {{"name": "accel (g)", "quantity": "acceleration", "unit": "g"}}
430  ]
431}}"#
432        )
433    }
434
435    fn message(error: LogError) -> String {
436        match error {
437            LogError::Mapping { message } | LogError::Layout { message } => message,
438            other => panic!("not a mapping's or layout's refusal: {other}"),
439        }
440    }
441
442    #[test]
443    fn a_marked_primary_track_is_the_records_altitude_on_its_clock() {
444        let mapping = mapping_from_json(&mapping("1")).unwrap();
445        let log = read_by_mapping(TABLE.as_bytes(), &mapping, None).unwrap();
446        assert_eq!(log.time_s, [0.0, 0.05, 0.1, 0.15]);
447        let feet = [0.0, 1.5, 6.0, 13.4].map(|ft: f64| ft * 0.3048);
448        assert_eq!(log.altitude_m, feet);
449        // The track stays a channel too, so the record keeps every sample.
450        let altitude: Vec<_> = log.channels_of(Quantity::AltitudeAboveGround).collect();
451        assert_eq!(altitude.len(), 1);
452        assert_eq!(altitude[0].values, feet);
453        assert_eq!(log.channels.len(), 2);
454    }
455
456    #[test]
457    fn a_mapping_that_marks_no_primary_track_leaves_it_empty() {
458        for primary in ["null", "none"] {
459            let text = if primary == "none" {
460                mapping("null").replace("  \"primary\": null,\n", "")
461            } else {
462                mapping(primary)
463            };
464            let mapping = mapping_from_json(&text).unwrap();
465            assert_eq!(mapping.primary, None);
466            let log = read_by_mapping(TABLE.as_bytes(), &mapping, None).unwrap();
467            assert!(log.time_s.is_empty() && log.altitude_m.is_empty());
468            assert_eq!(log.channels.len(), 2);
469        }
470    }
471
472    #[test]
473    fn a_primary_track_that_isnt_an_altitude_above_the_pad_is_refused() {
474        let refused = |primary: &str, table: &str| {
475            let mapping = mapping_from_json(&mapping(primary)).unwrap();
476            message(read_by_mapping(table.as_bytes(), &mapping, None).unwrap_err())
477        };
478        assert_eq!(refused("3", TABLE), "the primary track is column 3, of 3");
479        assert_eq!(
480            refused("0", TABLE),
481            "the primary track, \"t (ms)\", is the clock's column"
482        );
483        assert_eq!(
484            refused("2", TABLE),
485            "the primary track, \"accel (g)\", is mapped to `acceleration`: it must be \
486             `altitude_above_ground`, the barometric altitude above the pad"
487        );
488        let skipped = mapping("1").replace(r#""unit": "ft"}"#, r#""unit": "ft", "skip": true}"#);
489        let mapping_skipped = mapping_from_json(&skipped).unwrap();
490        assert_eq!(
491            message(read_by_mapping(TABLE.as_bytes(), &mapping_skipped, None).unwrap_err()),
492            "the primary track, \"alt (ft)\", is a column the mapping leaves out"
493        );
494        let kept = mapping("1").replace(
495            r#""quantity": "altitude_above_ground", "unit": "ft""#,
496            r#""quantity": null, "unit": null"#,
497        );
498        let mapping_kept = mapping_from_json(&kept).unwrap();
499        assert_eq!(
500            message(read_by_mapping(TABLE.as_bytes(), &mapping_kept, None).unwrap_err()),
501            "the primary track, \"alt (ft)\", is kept as written: it must be \
502             `altitude_above_ground`, the barometric altitude above the pad"
503        );
504        // A height its unit makes impossible keeps the column as written, so no track.
505        let impossible = TABLE.replace("100,6.0,6.1", "100,999999999,6.1");
506        assert_eq!(
507            refused("1", &impossible),
508            "the primary track, \"alt (ft)\", holds values that contradict its unit, \"ft\""
509        );
510    }
511
512    #[test]
513    fn a_primary_track_needs_altitudes_on_a_clock_that_goes_forward() {
514        let mapping = mapping_from_json(&mapping("1")).unwrap();
515        let empty = "t (ms),alt (ft),accel (g)\n0,,1.0\n50,,6.2\n100,,6.1\n";
516        assert_eq!(
517            message(read_by_mapping(empty.as_bytes(), &mapping, None).unwrap_err()),
518            "the primary track, \"alt (ft)\", holds no number"
519        );
520        // A clock that steps back is kept in the file's order as a channel's, but a track's
521        // times must go forward; a repeated time doesn't either.
522        for (back, line) in [("100,6.0", 4), ("50,6.0", 4)] {
523            let table = TABLE.replace("150,13.4", back);
524            let error = message(read_by_mapping(table.as_bytes(), &mapping, None).unwrap_err());
525            assert!(
526                error.starts_with(&format!(
527                    "the primary track, \"alt (ft)\", needs a clock that goes forward, and the \
528                     clock, \"t (ms)\", doesn't on line {}",
529                    line + 1
530                )),
531                "{error}"
532            );
533        }
534        // Without a primary track, the same clock reads, its step back noted.
535        let unmarked = mapping_from_json(&super::tests::mapping("null")).unwrap();
536        let table = TABLE.replace("150,13.4", "50,6.0");
537        let log = read_by_mapping(table.as_bytes(), &unmarked, None).unwrap();
538        assert!(log.notes.iter().any(|n| n.contains("steps back 1 times")));
539    }
540
541    #[test]
542    fn a_mapping_file_is_refused_naming_the_field_it_fails() {
543        let refused = |text: &str| message(mapping_from_json(text).unwrap_err());
544        let misspelled = mapping("1").replace("\"separator\"", "\"seperator\"");
545        assert!(
546            refused(&misspelled).starts_with("unknown field `seperator`, expected one of"),
547            "{}",
548            refused(&misspelled)
549        );
550        let missing = mapping("1").replace("  \"clock\": 0,\n", "");
551        assert!(refused(&missing).starts_with("missing field `clock`"));
552        let wrong = mapping("1").replace(r#""unit": "ft""#, r#""unit": 3"#);
553        assert!(
554            refused(&wrong).starts_with("`columns[1].unit`: invalid type: integer `3`"),
555            "{}",
556            refused(&wrong)
557        );
558        assert!(
559            refused(&wrong).ends_with("line 9 column 71"),
560            "{}",
561            refused(&wrong)
562        );
563        let column_key = mapping("1").replace(r#""unit": "ft""#, r#""unit": "ft", "units": "ft""#);
564        assert!(refused(&column_key).starts_with("`columns[1]`: unknown field `units`"));
565        let quantity = mapping("1").replace("acceleration", "accel");
566        assert!(
567            refused(&quantity).starts_with("`columns[2].quantity`: unknown variant `accel`"),
568            "{}",
569            refused(&quantity)
570        );
571        assert!(refused(&mapping("-1")).starts_with("`primary`: invalid value: integer `-1`"));
572        assert!(refused(&format!("{} {{}}", mapping("1"))).starts_with("trailing characters"));
573        let long = format!("{}{}", mapping("1"), " ".repeat(MAX_FILE_BYTES));
574        assert!(refused(&long).starts_with("a mapping file is at most 1048576 bytes"));
575        // A refused value as long as the file is quoted cut, not whole.
576        let huge = mapping("1").replace("acceleration", &"x".repeat(100_000));
577        let said = refused(&huge);
578        assert!(
579            said.starts_with("`columns[2].quantity`: unknown variant `xxx"),
580            "{said}"
581        );
582        assert!(said.ends_with('…'), "{said}");
583        assert_eq!(
584            said.chars().count(),
585            "`columns[2].quantity`: ".chars().count() + MAX_WORDS_CHARS
586        );
587    }
588
589    #[test]
590    fn a_files_text_is_utf8_or_utf16_with_its_mark() {
591        let text = mapping("1");
592        assert_eq!(json_text(text.as_bytes()), Some(text.clone()));
593        let marked = [&[0xEF, 0xBB, 0xBF][..], text.as_bytes()].concat();
594        assert_eq!(json_text(&marked), Some(text.clone()));
595        let little: Vec<u8> = [0xFF, 0xFE]
596            .into_iter()
597            .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
598            .collect();
599        assert_eq!(json_text(&little), Some(text.clone()));
600        let big: Vec<u8> = [0xFE, 0xFF]
601            .into_iter()
602            .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
603            .collect();
604        assert_eq!(json_text(&big), Some(text.clone()));
605        // Read, each is the mapping the text states.
606        assert_eq!(
607            mapping_from_json(&json_text(&little).unwrap()).unwrap(),
608            mapping_from_json(&text).unwrap()
609        );
610        assert_eq!(json_text(&little[..little.len() - 1]), None);
611        assert_eq!(json_text(&little[2..]), None, "UTF-16 with no mark");
612        assert_eq!(json_text(&big[2..]), None, "UTF-16 with no mark");
613        assert_eq!(json_text(&[0xFF, 0xFE, 0x00, 0xD8]), None);
614        assert_eq!(json_text(b"{\xff}"), None);
615    }
616
617    const LAYOUT: &str = r#"{
618  "fields": [
619    {"name": "time__ms", "kind": "u32"},
620    {"name": "pressure__pa", "kind": "f32"}
621  ]
622}"#;
623
624    /// Three records of `LAYOUT`, little-endian.
625    fn records() -> Vec<u8> {
626        let mut bytes = Vec::new();
627        for (time, pressure) in [(0_u32, 101_325.0_f32), (20, 101_300.0), (40, 101_250.0)] {
628            bytes.extend(time.to_le_bytes());
629            bytes.extend(pressure.to_le_bytes());
630        }
631        bytes
632    }
633
634    #[test]
635    fn a_layout_file_reads_records_as_the_library_does() {
636        let layout = layout_from_json(LAYOUT).unwrap();
637        assert_eq!(layout.byte_order, ByteOrder::Little);
638        assert_eq!(layout.block, None);
639        let log = read_by_layout(&records(), &layout).unwrap();
640        assert_eq!(
641            log,
642            records::read(&records(), &layout.record_layout()).unwrap()
643        );
644        assert_eq!(log.channels[0].values, [101_325.0, 101_300.0, 101_250.0]);
645    }
646
647    #[test]
648    fn a_layout_reads_a_captures_block_it_names_and_refuses_one_it_doesnt() {
649        let encoded = base64(&records());
650        let capture = format!(
651            "uart:~$ log dump\n----++++//[[( data start )]]\\\\++++----\n{encoded}\n\
652             ----++++//[[( data end )]]\\\\++++----\nuart:~$\n"
653        );
654        assert!(capture::recognizes(&capture), "{capture}");
655        let named = LAYOUT.replace("\n  ]\n}", "\n  ],\n  \"block\": \"data\"\n}");
656        let layout = layout_from_json(&named).unwrap();
657        let log = read_by_layout(capture.as_bytes(), &layout).unwrap();
658        assert_eq!(
659            log,
660            capture::read(&capture, "data", &layout.record_layout()).unwrap()
661        );
662        let unnamed = layout_from_json(LAYOUT).unwrap();
663        assert_eq!(
664            message(read_by_layout(capture.as_bytes(), &unnamed).unwrap_err()),
665            "the file is a serial console's capture of records as base64, its blocks \"data\". \
666             The layout must name the block its records are in, as `block`"
667        );
668        assert!(
669            message(read_by_layout(&records(), &layout).unwrap_err())
670                .starts_with("the layout names the block \"data\" of a serial console's capture")
671        );
672    }
673
674    #[test]
675    fn a_layout_file_is_refused_naming_the_field_it_fails() {
676        let refused = |text: &str| message(layout_from_json(text).unwrap_err());
677        let kind = LAYOUT.replace("\"f32\"", "\"float\"");
678        assert!(
679            refused(&kind).starts_with("`fields[1].kind`: unknown variant `float`"),
680            "{}",
681            refused(&kind)
682        );
683        let key = LAYOUT.replace("\"fields\"", "\"field\"");
684        assert!(refused(&key).starts_with("unknown field `field`"));
685        let order = LAYOUT.replace("\n  ]\n}", "\n  ],\n  \"byte_order\": \"middle\"\n}");
686        assert!(refused(&order).starts_with("`byte_order`: unknown variant `middle`"));
687    }
688
689    #[test]
690    fn a_sheet_named_for_a_text_file_is_refused() {
691        let mapping = mapping_from_json(&mapping("1")).unwrap();
692        let error = message(read_by_mapping(TABLE.as_bytes(), &mapping, Some(0)).unwrap_err());
693        assert!(
694            error.starts_with("a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook"),
695            "{error}"
696        );
697    }
698
699    /// Standard base64 with padding, as a console capture writes its records.
700    fn base64(bytes: &[u8]) -> String {
701        const ALPHABET: &[u8; 64] =
702            b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
703        let mut text = String::new();
704        for chunk in bytes.chunks(3) {
705            let word = chunk.iter().enumerate().fold(0_u32, |word, (i, byte)| {
706                word | u32::from(*byte) << (16 - 8 * i)
707            });
708            for i in 0..4 {
709                if i <= chunk.len() {
710                    text.push(char::from(ALPHABET[(word >> (18 - 6 * i) & 63) as usize]));
711                } else {
712                    text.push('=');
713                }
714            }
715        }
716        text
717    }
718}