Skip to main content

hpr_flightdata/
records.rs

1//! Binary records read by a layout the caller states: a team's flight computer that writes each
2//! sample as a fixed run of bytes, such as a `u32` clock in milliseconds then 24 `f32` readings,
3//! with nothing in the file to say so.
4//!
5//! No maker documents these files, and their bytes carry no names: a record of a clock and two
6//! readings looks like any other 12 bytes. So the reader takes the layout from the caller, each
7//! field's name and type in the record's order ([`RecordLayout`]), and never guesses one. The
8//! names do the work a table's header does: the records are written out as a table, the names its
9//! header and one row a record, and the table is read by [`generic::read`], which reads a column
10//! in a unit only where its name states one, such as `timestamp__ms` or `Time (ms)`, and keeps
11//! every other as written. A layout whose names state no clock is refused; [`table`] gives the
12//! text, for a caller who would rather read it by a column mapping ([`generic::read_with`]).
13//!
14//! **Records.** The file is records end to end, from its first byte, each the layout's fields with
15//! no gap between them, in little-endian byte order unless the layout says big. A field is an
16//! unsigned or signed integer of 8, 16, 32 or 64 bits, or an IEEE 754 binary32 or binary64 float
17//! ([`FieldKind`]). A float that isn't finite (a NaN or an infinity) is written as an empty cell,
18//! which the table's reader keeps as a gap, and counted in a note.
19//!
20//! **Erased flash.** A logger that saves to flash memory leaves the memory it never wrote erased,
21//! every byte `0xFF`, and a dump of the memory may run past its last record into it. A run of
22//! records each wholly `0xFF` at the file's end is that memory, not samples: it is dropped, and a
23//! note counts it. So are bytes past the last whole record if every one is `0xFF`. Any other bytes
24//! past the last whole record are refused, as a layout that doesn't fit the file. An erased record
25//! with a written one after it is read as written: its unsigned integers at their largest, its
26//! signed ones −1, its floats NaN. So a file with a float clock is refused at such a record, its
27//! clock no number; with an integer clock, the record's time and values stay in the log, and a
28//! column whose values it pushes past what its unit allows is kept as written.
29//!
30//! **Bounds.** A layout has from 2 to [`MAX_FIELDS`] fields, each named with 1 to
31//! [`MAX_NAME_CHARS`] characters, no two alike, none a number or holding a separator (a comma, a
32//! semicolon or `|`), a quotation mark, a control character or spaces at its ends, and the first
33//! not opening with `#`, `sep=` or a byte order mark. The table is refused once its text would pass [`MAX_TEXT_BYTES`].
34//!
35//! How far to trust it: the reader is exact for the bytes and types the caller states; whether the
36//! layout is the logger's is the caller's statement. The project's sample files are each held,
37//! field by field, to the table the logger's team decoded from the same bytes (`cargo xtask
38//! logs`, [ADR-271][adr-271]).
39//!
40//! [adr-271]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/docs/decisions/0271-binary-records-by-a-stated-layout-before-the-rest.md
41//!
42//! ```
43//! use hpr_flightdata::records::{self, Field, FieldKind, RecordLayout};
44//!
45//! // Two invented records: a clock in milliseconds and a pressure in kilopascals.
46//! let mut bytes = Vec::new();
47//! for (ms, kpa) in [(1000_u32, 101.3_f32), (1010, 101.2)] {
48//!     bytes.extend(ms.to_le_bytes());
49//!     bytes.extend(kpa.to_le_bytes());
50//! }
51//! let layout = RecordLayout::new(vec![
52//!     Field::new("time__ms", FieldKind::U32),
53//!     Field::new("pressure__kpa", FieldKind::F32),
54//! ]);
55//! let log = records::read(&bytes, &layout)?;
56//! assert_eq!(log.channels[0].time_s, [1.0, 1.01]);
57//! assert_eq!(log.channels[0].values[0], f64::from(101.3_f32) * 1000.0); // pascals
58//! # Ok::<(), hpr_flightdata::error::LogError>(())
59//! ```
60
61use std::fmt::Write as _;
62
63use schemars::JsonSchema;
64use serde::{Deserialize, Serialize};
65
66use crate::error::LogError;
67use crate::generic;
68use crate::log::{FlightLog, LogFormat};
69use crate::table::short;
70
71/// The most fields a layout may have: as many columns as a table may.
72pub const MAX_FIELDS: usize = generic::MAX_COLUMNS;
73
74/// The most characters in a field's name.
75pub const MAX_NAME_CHARS: usize = 256;
76
77/// The most bytes of text the records may be written out as: a bound on the memory a file can make
78/// the reader take, whatever its layout (a binary64 can write over 300 characters).
79pub const MAX_TEXT_BYTES: usize = 256 * 1024 * 1024;
80
81/// The format's name, as its errors give it.
82const FORMAT: &str = "binary records";
83
84/// The type of one field of a record.
85#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
86#[serde(rename_all = "snake_case")]
87#[non_exhaustive]
88pub enum FieldKind {
89    /// An unsigned 8-bit integer.
90    U8,
91    /// A signed 8-bit integer, two's complement.
92    I8,
93    /// An unsigned 16-bit integer.
94    U16,
95    /// A signed 16-bit integer, two's complement.
96    I16,
97    /// An unsigned 32-bit integer.
98    U32,
99    /// A signed 32-bit integer, two's complement.
100    I32,
101    /// An unsigned 64-bit integer.
102    U64,
103    /// A signed 64-bit integer, two's complement.
104    I64,
105    /// An IEEE 754 binary32 float.
106    F32,
107    /// An IEEE 754 binary64 float.
108    F64,
109}
110
111impl FieldKind {
112    /// The field's size, in bytes.
113    pub fn bytes(self) -> usize {
114        match self {
115            Self::U8 | Self::I8 => 1,
116            Self::U16 | Self::I16 => 2,
117            Self::U32 | Self::I32 | Self::F32 => 4,
118            Self::U64 | Self::I64 | Self::F64 => 8,
119        }
120    }
121}
122
123/// The order of a field's bytes.
124#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
125#[serde(rename_all = "snake_case")]
126pub enum ByteOrder {
127    /// The least significant byte first, as ARM and x86 processors store a number.
128    #[default]
129    Little,
130    /// The most significant byte first.
131    Big,
132}
133
134/// One field of a record: its name, as a table's header would write it, and its type.
135#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
136#[serde(deny_unknown_fields)]
137pub struct Field {
138    /// The field's name, its unit in it where it has one, such as `pressure__kpa` or `Time (ms)`
139    /// ([`generic`]'s *Units in a name*).
140    pub name: String,
141    /// The field's type.
142    pub kind: FieldKind,
143}
144
145impl Field {
146    /// A field of the name and type.
147    pub fn new(name: impl Into<String>, kind: FieldKind) -> Self {
148        Self {
149            name: name.into(),
150            kind,
151        }
152    }
153}
154
155/// How each record of a file is laid out: its fields, in order, with no gap between them.
156#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
157pub struct RecordLayout {
158    /// The fields, in the record's order.
159    pub fields: Vec<Field>,
160    /// The order of each field's bytes; little-endian if not stated.
161    #[serde(default)]
162    pub byte_order: ByteOrder,
163}
164
165impl RecordLayout {
166    /// A little-endian layout of the fields.
167    pub fn new(fields: Vec<Field>) -> Self {
168        Self {
169            fields,
170            byte_order: ByteOrder::Little,
171        }
172    }
173
174    /// A record's size, in bytes: its fields' sizes summed.
175    pub fn record_bytes(&self) -> usize {
176        self.fields.iter().map(|field| field.kind.bytes()).sum()
177    }
178}
179
180/// A file's records written out as a table, by [`table`].
181#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
182#[non_exhaustive]
183pub struct Table {
184    /// The table: a header row of the fields' names, then one row a record, comma-separated,
185    /// each line ended by LF.
186    pub text: String,
187    /// The records written.
188    pub records: usize,
189    /// A record's size, bytes: record `n`, from 1, starts at byte `(n − 1) × record_bytes`.
190    pub record_bytes: usize,
191    /// What the reader noticed: erased memory dropped, values that aren't finite.
192    pub notes: Vec<String>,
193}
194
195fn layout_error(message: String) -> LogError {
196    LogError::Layout { message }
197}
198
199/// Refuses a layout outside the bounds (module docs, *Bounds*).
200fn check(layout: &RecordLayout) -> Result<(), LogError> {
201    let count = layout.fields.len();
202    // A table's header names two columns at least: a clock and something on it.
203    if !(2..=MAX_FIELDS).contains(&count) {
204        return Err(layout_error(format!(
205            "a layout has from 2 to {MAX_FIELDS} fields; this one has {count}"
206        )));
207    }
208    let mut seen = std::collections::BTreeSet::new();
209    for (index, field) in layout.fields.iter().enumerate() {
210        let name = &field.name;
211        let chars = name.chars().count();
212        let why = if chars == 0 {
213            Some("has no name".to_owned())
214        } else if chars > MAX_NAME_CHARS {
215            Some(format!(
216                "has a name of more than {MAX_NAME_CHARS} characters"
217            ))
218        } else if name.contains([',', ';', '|', '"']) || name.chars().any(char::is_control) {
219            // The table reader would take `;` or `|` for the separator.
220            Some(format!(
221                "is named {}, holding a separator (a comma, a semicolon or `|`), a quotation \
222                 mark or a control character",
223                quoted(name)
224            ))
225        } else if index == 0
226            && (name.starts_with(['#', '\u{FEFF}'])
227                || name.to_ascii_lowercase().starts_with("sep="))
228        {
229            // The table reader skips a `#` comment, takes `sep=` for the separator's line and
230            // drops a byte order mark opening the text.
231            Some(format!(
232                "is named {}, which a table's first line can't open with",
233                quoted(name)
234            ))
235        } else if name.parse::<f64>().is_ok_and(f64::is_finite) {
236            // A header holds no number, so a table's reader doesn't take a row for its header.
237            Some(format!("is named {}, a number", quoted(name)))
238        } else if name.trim() != name {
239            Some(format!(
240                "is named {}, with spaces at its ends",
241                quoted(name)
242            ))
243        } else if !seen.insert(name.as_str()) {
244            Some(format!("is named {}, as an earlier field is", quoted(name)))
245        } else {
246            None
247        };
248        if let Some(why) = why {
249            return Err(layout_error(format!(
250                "field {} of the layout {why}",
251                index + 1
252            )));
253        }
254    }
255    Ok(())
256}
257
258fn quoted(name: &str) -> String {
259    format!("`{}`", short(name))
260}
261
262/// One field's value, from its bytes, written as the table's cell; `None` for a float that isn't
263/// finite.
264fn cell(kind: FieldKind, bytes: &[u8], order: ByteOrder, out: &mut String) -> bool {
265    macro_rules! read {
266        ($ty:ty) => {{
267            let mut word = [0u8; std::mem::size_of::<$ty>()];
268            word.copy_from_slice(bytes);
269            match order {
270                ByteOrder::Little => <$ty>::from_le_bytes(word),
271                ByteOrder::Big => <$ty>::from_be_bytes(word),
272            }
273        }};
274    }
275    // Writing to a String can't fail.
276    let float = |value: f64, out: &mut String| {
277        if value.is_finite() {
278            let _ = write!(out, "{value}");
279            true
280        } else {
281            false
282        }
283    };
284    match kind {
285        FieldKind::U8 => {
286            let _ = write!(out, "{}", read!(u8));
287        }
288        FieldKind::I8 => {
289            let _ = write!(out, "{}", read!(i8));
290        }
291        FieldKind::U16 => {
292            let _ = write!(out, "{}", read!(u16));
293        }
294        FieldKind::I16 => {
295            let _ = write!(out, "{}", read!(i16));
296        }
297        FieldKind::U32 => {
298            let _ = write!(out, "{}", read!(u32));
299        }
300        FieldKind::I32 => {
301            let _ = write!(out, "{}", read!(i32));
302        }
303        FieldKind::U64 => {
304            let _ = write!(out, "{}", read!(u64));
305        }
306        FieldKind::I64 => {
307            let _ = write!(out, "{}", read!(i64));
308        }
309        // A binary32 widens to binary64 exactly, and `{}` writes the shortest decimal that reads
310        // back to the same binary64, so the table's number is the field's value.
311        FieldKind::F32 => return float(f64::from(read!(f32)), out),
312        FieldKind::F64 => return float(read!(f64), out),
313    }
314    true
315}
316
317/// The file's records written out as a table (module docs); [`table_within`] with
318/// [`MAX_TEXT_BYTES`].
319///
320/// # Errors
321///
322/// As [`table_within`].
323pub fn table(bytes: &[u8], layout: &RecordLayout) -> Result<Table, LogError> {
324    table_within(bytes, layout, MAX_TEXT_BYTES)
325}
326
327/// The file's records written out as a table of at most `max_text_bytes` (module docs).
328///
329/// # Errors
330///
331/// [`LogError::Layout`] for a layout outside the bounds, or a table whose text would pass
332/// `max_text_bytes`; [`LogError::Binary`] for bytes past the last whole record that aren't all
333/// `0xFF`, naming the first; [`LogError::NoData`] for a file with no record but erased memory.
334pub fn table_within(
335    bytes: &[u8],
336    layout: &RecordLayout,
337    max_text_bytes: usize,
338) -> Result<Table, LogError> {
339    check(layout)?;
340    let size = layout.record_bytes();
341    let whole = bytes.len() / size;
342    let tail = &bytes[whole * size..];
343    let mut notes = Vec::new();
344    if !tail.is_empty() {
345        if let Some(at) = tail.iter().position(|&byte| byte != 0xFF) {
346            return Err(LogError::Binary {
347                format: FORMAT,
348                offset: whole * size + at,
349                message: format!(
350                    "the file's {} bytes aren't whole records of {size} bytes: {} are left past \
351                     the last, and they aren't erased memory, all 0xFF",
352                    bytes.len(),
353                    tail.len()
354                ),
355            });
356        }
357        notes.push(format!(
358            "{} bytes past the last whole record, each 0xFF, dropped as erased memory",
359            tail.len()
360        ));
361    }
362    // The records are walked, never collected: a list of them would take 16 bytes a record.
363    let body = &bytes[..whole * size];
364    let erased = body
365        .rchunks_exact(size)
366        .take_while(|record| record.iter().all(|&byte| byte == 0xFF))
367        .count();
368    if erased > 0 {
369        notes.push(format!(
370            "the last {erased} of {whole} records are erased memory, every byte 0xFF: dropped"
371        ));
372    }
373    let written = whole - erased;
374    if written == 0 {
375        return Err(LogError::NoData { format: FORMAT });
376    }
377    // Each record writes a separator a field at least, so a table past the bound is refused
378    // before any of it is written.
379    let fewest = written.saturating_mul(layout.fields.len());
380    if fewest > max_text_bytes {
381        return Err(layout_error(format!(
382            "the records, written out as a table, would pass {max_text_bytes} bytes: its \
383             {written} records write {fewest} at least"
384        )));
385    }
386    let names: Vec<&str> = layout.fields.iter().map(|f| f.name.as_str()).collect();
387    let mut text = names.join(",");
388    text.push('\n');
389    let mut not_finite = 0usize;
390    for record in body.chunks_exact(size).take(written) {
391        let mut at = 0;
392        for (index, field) in layout.fields.iter().enumerate() {
393            if index > 0 {
394                text.push(',');
395            }
396            let width = field.kind.bytes();
397            if !cell(
398                field.kind,
399                &record[at..at + width],
400                layout.byte_order,
401                &mut text,
402            ) {
403                not_finite += 1;
404            }
405            at += width;
406        }
407        text.push('\n');
408        if text.len() > max_text_bytes {
409            return Err(layout_error(format!(
410                "the records, written out as a table, would pass {max_text_bytes} bytes"
411            )));
412        }
413    }
414    if not_finite > 0 {
415        notes.push(format!(
416            "{not_finite} float values aren't finite, NaN or infinite: kept as gaps"
417        ));
418    }
419    Ok(Table {
420        text,
421        records: written,
422        record_bytes: size,
423        notes,
424    })
425}
426
427/// Reads a file of binary records by the layout the caller states (module docs), as
428/// [`LogFormat::BinaryRecords`].
429///
430/// # Errors
431///
432/// As [`table`]; [`LogError::Layout`] if the layout's names state no clock with a unit of time,
433/// such as `Time (ms)`; [`LogError::Binary`], naming the record and its offset, for a record the
434/// table's reader refuses, such as a clock that isn't a number; otherwise as [`generic::read`]
435/// refuses the table, such as a clock that never steps forward.
436pub fn read(bytes: &[u8], layout: &RecordLayout) -> Result<FlightLog, LogError> {
437    let table = table(bytes, layout)?;
438    let mut log = read_table(&table)?;
439    log.format = LogFormat::BinaryRecords;
440    Ok(log)
441}
442
443/// The table read by its header, the records' notes before the table's.
444pub(crate) fn read_table(table: &Table) -> Result<FlightLog, LogError> {
445    let mut log = generic::read(&table.text).map_err(|error| match error {
446        LogError::NotThisFormat { .. } => layout_error(
447            "the layout's names state no clock with a unit of time, such as `Time (ms)` or \
448             `timestamp__us`. Name the clock so, or read the records' table by a column mapping \
449             (`records::table`, then `generic::read_with`)"
450                .to_owned(),
451        ),
452        // The table's line `n` is record `n − 1`: name the record, not a line the caller never saw.
453        LogError::Syntax { line, message, .. } if line >= 2 => LogError::Binary {
454            format: FORMAT,
455            offset: (line - 2).saturating_mul(table.record_bytes),
456            message: format!("record {}: {message}", line - 1),
457        },
458        other => other,
459    })?;
460    let mut notes = vec![format!(
461        "{} binary records read by the caller's layout, written out as a table and read by its \
462         header",
463        table.records
464    )];
465    notes.extend(table.notes.iter().cloned());
466    notes.append(&mut log.notes);
467    log.notes = notes;
468    Ok(log)
469}
470
471#[cfg(test)]
472mod tests {
473    use super::*;
474    use crate::log::Quantity;
475
476    fn layout(fields: &[(&str, FieldKind)]) -> RecordLayout {
477        RecordLayout::new(
478            fields
479                .iter()
480                .map(|(name, kind)| Field::new(*name, *kind))
481                .collect(),
482        )
483    }
484
485    fn clock_and_pressure() -> RecordLayout {
486        layout(&[
487            ("time__ms", FieldKind::U32),
488            ("pressure__kpa", FieldKind::F32),
489        ])
490    }
491
492    fn records(rows: &[(u32, f32)]) -> Vec<u8> {
493        rows.iter()
494            .flat_map(|(ms, kpa)| ms.to_le_bytes().into_iter().chain(kpa.to_le_bytes()))
495            .collect()
496    }
497
498    #[test]
499    fn each_record_is_a_row_its_names_read_in_their_units() {
500        let bytes = records(&[(1000, 101.3), (1010, 101.25), (1020, 101.2)]);
501        let log = read(&bytes, &clock_and_pressure()).unwrap();
502        assert_eq!(log.format, LogFormat::BinaryRecords);
503        assert_eq!(log.channels[0].time_s, [1.0, 1.01, 1.02]);
504        let pressure = &log.channels[0];
505        assert_eq!(pressure.column, "pressure__kpa");
506        assert_eq!(pressure.quantity, Quantity::Pressure);
507        // The binary32 widened exactly, then kilopascals to pascals.
508        assert_eq!(pressure.values[1], f64::from(101.25_f32) * 1000.0);
509        assert!(
510            log.notes[0].starts_with("3 binary records"),
511            "{:?}",
512            log.notes
513        );
514    }
515
516    #[test]
517    fn every_kind_is_read_at_its_width_in_either_byte_order() {
518        let kinds = [
519            (FieldKind::U8, vec![0xFE], "254"),
520            (FieldKind::I8, vec![0xFE], "-2"),
521            (FieldKind::U16, vec![0x01, 0x02], "513"),
522            (FieldKind::I16, vec![0xFF, 0xFE], "-257"),
523            (FieldKind::U32, vec![0x01, 0, 0, 0x80], "2147483649"),
524            (FieldKind::I32, vec![0xFF, 0xFF, 0xFF, 0xFF], "-1"),
525            (FieldKind::U64, vec![0xFF; 8], "18446744073709551615"),
526            (
527                FieldKind::I64,
528                vec![0, 0, 0, 0, 0, 0, 0, 0x80],
529                "-9223372036854775808",
530            ),
531            (
532                FieldKind::F32,
533                0.1_f32.to_le_bytes().to_vec(),
534                "0.10000000149011612",
535            ),
536            (FieldKind::F64, 0.1_f64.to_le_bytes().to_vec(), "0.1"),
537        ];
538        for (kind, little, written) in kinds {
539            assert_eq!(kind.bytes(), little.len(), "{kind:?}");
540            let mut out = String::new();
541            assert!(cell(kind, &little, ByteOrder::Little, &mut out));
542            assert_eq!(out, written, "{kind:?} little-endian");
543            let big: Vec<u8> = little.iter().rev().copied().collect();
544            let mut out = String::new();
545            assert!(cell(kind, &big, ByteOrder::Big, &mut out));
546            assert_eq!(out, written, "{kind:?} big-endian");
547        }
548    }
549
550    /// Values at the edges of their types survive the table exactly: the largest and smallest
551    /// binary64, a subnormal of each width, negative zero, and integers at their ends.
552    #[test]
553    fn extreme_values_survive_the_table_exactly() {
554        let layout = layout(&[
555            ("time__ms", FieldKind::U32),
556            ("a", FieldKind::F64),
557            ("b", FieldKind::F32),
558            ("c", FieldKind::I64),
559            ("d", FieldKind::U64),
560        ]);
561        let rows: [(u32, f64, f32, i64, u64); 4] = [
562            (0, f64::MAX, f32::MAX, i64::MIN, u64::MAX),
563            (10, f64::from_bits(1), f32::from_bits(1), i64::MAX, 0),
564            (20, -0.0, -0.0, -1, 1),
565            (30, f64::MIN_POSITIVE, f32::MIN_POSITIVE, 0, 1 << 53),
566        ];
567        let mut bytes = Vec::new();
568        for (t, a, b, c, d) in rows {
569            bytes.extend(t.to_le_bytes());
570            bytes.extend(a.to_le_bytes());
571            bytes.extend(b.to_le_bytes());
572            bytes.extend(c.to_le_bytes());
573            bytes.extend(d.to_le_bytes());
574        }
575        let log = read(&bytes, &layout).unwrap();
576        let column = |name: &str| {
577            &log.channels
578                .iter()
579                .find(|channel| channel.column == name)
580                .unwrap()
581                .values
582        };
583        for (i, (_, a, b, _, _)) in rows.iter().enumerate() {
584            assert_eq!(column("a")[i].to_bits(), a.to_bits(), "row {i}");
585            assert_eq!(column("b")[i].to_bits(), f64::from(*b).to_bits(), "row {i}");
586        }
587        // A 64-bit integer is written in full; read as a number, it is the nearest binary64.
588        let table = table(&bytes, &layout).unwrap();
589        assert!(
590            table
591                .text
592                .contains(",-9223372036854775808,18446744073709551615\n")
593        );
594        assert_eq!(column("d")[3], 9_007_199_254_740_992.0);
595    }
596
597    #[test]
598    fn a_big_endian_layout_reads_its_records() {
599        let mut bytes = Vec::new();
600        for (ms, kpa) in [(1000_u32, 101.3_f32), (1010, 101.2)] {
601            bytes.extend(ms.to_be_bytes());
602            bytes.extend(kpa.to_be_bytes());
603        }
604        let mut layout = clock_and_pressure();
605        layout.byte_order = ByteOrder::Big;
606        assert_eq!(
607            read(&bytes, &layout).unwrap().channels[0].time_s,
608            [1.0, 1.01]
609        );
610        // Read little-endian, the same bytes are another clock.
611        layout.byte_order = ByteOrder::Little;
612        assert_ne!(
613            read(&bytes, &layout).map(|log| log.channels[0].time_s.clone()),
614            Ok(vec![1.0, 1.01])
615        );
616    }
617
618    #[test]
619    fn a_float_that_isnt_finite_is_a_gap_and_counted() {
620        let bytes = records(&[(1000, 101.3), (1010, f32::NAN), (1020, f32::INFINITY)]);
621        let log = read(&bytes, &clock_and_pressure()).unwrap();
622        let values = &log.channels[0].values;
623        assert!(values[1].is_nan() && values[2].is_nan(), "{values:?}");
624        assert!(
625            log.notes
626                .iter()
627                .any(|n| n.starts_with("2 float values aren't finite")),
628            "{:?}",
629            log.notes
630        );
631    }
632
633    #[test]
634    fn trailing_erased_records_and_bytes_are_dropped_and_counted() {
635        let mut bytes = records(&[(1000, 101.3), (1010, 101.2)]);
636        bytes.extend([0xFF; 8 * 3 + 5]);
637        let table = table(&bytes, &clock_and_pressure()).unwrap();
638        assert_eq!(table.records, 2);
639        assert_eq!(table.text.lines().count(), 3);
640        assert_eq!(
641            table.notes,
642            [
643                "5 bytes past the last whole record, each 0xFF, dropped as erased memory",
644                "the last 3 of 5 records are erased memory, every byte 0xFF: dropped",
645            ]
646        );
647    }
648
649    #[test]
650    fn an_erased_record_before_a_written_one_is_read_as_written() {
651        let mut bytes = records(&[(1000, 101.3), (1010, 101.3), (1020, 101.2)]);
652        bytes.extend([0xFF; 8]);
653        bytes.extend(records(&[(1040, 101.2), (1050, 101.1)]));
654        let table = table(&bytes, &clock_and_pressure()).unwrap();
655        assert_eq!(table.records, 6);
656        assert!(table.text.contains("\n4294967295,\n"), "{}", table.text);
657        // Its clock at its largest, its float a gap, as the logger's bytes say.
658        let log = read(&bytes, &clock_and_pressure()).unwrap();
659        let pressure = &log.channels[0];
660        assert_eq!(pressure.time_s[3], 4_294_967.295);
661        assert!(pressure.values[3].is_nan());
662        assert_eq!(pressure.time_s[4], 1.04);
663    }
664
665    #[test]
666    fn bytes_past_the_last_record_that_arent_erased_are_refused_at_the_first() {
667        let mut bytes = records(&[(1000, 101.3), (1010, 101.2)]);
668        bytes.extend([0xFF, 0xFF, 0x00]);
669        match table(&bytes, &clock_and_pressure()) {
670            Err(LogError::Binary {
671                offset, message, ..
672            }) => {
673                assert_eq!(offset, 18);
674                assert!(message.contains("3 are left past the last"), "{message}");
675            }
676            other => panic!("expected a binary refusal, got {other:?}"),
677        }
678    }
679
680    #[test]
681    fn a_file_of_only_erased_memory_has_no_data() {
682        for bytes in [vec![], vec![0xFF; 16], vec![0xFF; 3]] {
683            assert_eq!(
684                table(&bytes, &clock_and_pressure()),
685                Err(LogError::NoData { format: FORMAT }),
686                "{} bytes",
687                bytes.len()
688            );
689        }
690    }
691
692    #[test]
693    fn a_layout_outside_the_bounds_is_refused_naming_why() {
694        let many: Vec<(String, FieldKind)> = (0..=MAX_FIELDS)
695            .map(|i| (format!("f{i}"), FieldKind::U8))
696            .collect();
697        let many = RecordLayout::new(
698            many.into_iter()
699                .map(|(name, kind)| Field::new(name, kind))
700                .collect(),
701        );
702        let long = "x".repeat(MAX_NAME_CHARS + 1);
703        // A clock, then a field of the name.
704        let named = |name: &str| layout(&[("t (ms)", FieldKind::U8), (name, FieldKind::U8)]);
705        let first = |name: &str| layout(&[(name, FieldKind::U8), ("t (ms)", FieldKind::U8)]);
706        let cases = [
707            (layout(&[]), "from 2 to 512 fields; this one has 0"),
708            (layout(&[("t (ms)", FieldKind::U8)]), "this one has 1"),
709            (many, "this one has 513"),
710            (named(""), "field 2 of the layout has no name"),
711            (named(&long), "more than 256 characters"),
712            (
713                named("a,b"),
714                "holding a separator (a comma, a semicolon or `|`)",
715            ),
716            (named("a;b"), "holding a separator"),
717            (named("a|b"), "holding a separator"),
718            (named("a\"b"), "a quotation mark"),
719            (named("a\nb"), "a control character"),
720            (named("a\tb"), "a control character"),
721            (named(" a"), "spaces at its ends"),
722            (named("12"), "field 2 of the layout is named `12`, a number"),
723            (named("1e3"), "a number"),
724            (
725                first("#t (ms)"),
726                "field 1 of the layout is named `#t (ms)`, which a table's first line can't open with",
727            ),
728            (first("SEP=x"), "can't open with"),
729            (first("\u{FEFF}t (ms)"), "can't open with"),
730            (
731                named("t (ms)"),
732                "field 2 of the layout is named `t (ms)`, as an earlier",
733            ),
734        ];
735        for (layout, why) in cases {
736            match table(&[0; 64], &layout) {
737                Err(LogError::Layout { message }) => {
738                    assert!(message.contains(why), "{message} lacks {why}");
739                }
740                other => panic!("expected a layout refusal for {why}, got {other:?}"),
741            }
742        }
743        // A name of exactly the most characters is a name; so is `#` past the first field, and a
744        // word that only parses as a number that isn't finite.
745        let most = "t".repeat(MAX_NAME_CHARS);
746        for layout in [named(&most), named("#2"), named("nan"), named("inf")] {
747            assert!(table(&[0; 4], &layout).is_ok(), "{layout:?}");
748        }
749    }
750
751    #[test]
752    fn names_that_state_no_clock_are_refused_pointing_to_a_mapping() {
753        let bytes = records(&[(1000, 101.3), (1010, 101.2)]);
754        let unnamed = layout(&[("col1", FieldKind::U32), ("col2", FieldKind::F32)]);
755        match read(&bytes, &unnamed) {
756            Err(LogError::Layout { message }) => {
757                assert!(message.contains("generic::read_with"), "{message}");
758            }
759            other => panic!("expected a layout refusal, got {other:?}"),
760        }
761        // The same records, read by a mapping of their table.
762        let table = table(&bytes, &unnamed).unwrap();
763        let mut mapping = generic::mapping(&table.text.replace("col1", "time (ms)")).unwrap();
764        mapping.columns[0].name = "col1".to_owned();
765        let log = generic::read_with(&table.text, &mapping).unwrap();
766        assert_eq!(log.channels[0].time_s, [1.0, 1.01]);
767    }
768
769    #[test]
770    fn a_table_past_its_bound_is_refused() {
771        let bytes = records(&[(1000, 101.3), (1010, 101.2)]);
772        let text = table(&bytes, &clock_and_pressure()).unwrap().text;
773        let exact = table_within(&bytes, &clock_and_pressure(), text.len()).unwrap();
774        assert_eq!(exact.text, text);
775        match table_within(&bytes, &clock_and_pressure(), text.len() - 1) {
776            Err(LogError::Layout { message }) => {
777                assert!(
778                    message.contains(&format!("would pass {} bytes", text.len() - 1)),
779                    "{message}"
780                );
781            }
782            other => panic!("expected the bound, got {other:?}"),
783        }
784    }
785
786    #[test]
787    fn a_table_that_cant_fit_is_refused_before_it_is_written() {
788        // Two fields a record write 2 bytes at least: 1,000 records can't fit in 1,999.
789        let bytes = vec![0u8; 2 * 1000];
790        let layout = layout(&[("t (ms)", FieldKind::U8), ("a", FieldKind::U8)]);
791        match table_within(&bytes, &layout, 1999) {
792            Err(LogError::Layout { message }) => {
793                assert!(
794                    message
795                        .ends_with("would pass 1999 bytes: its 1000 records write 2000 at least")
796                )
797            }
798            other => panic!("expected the bound, got {other:?}"),
799        }
800    }
801
802    #[test]
803    fn a_record_the_tables_reader_refuses_is_named_with_its_offset() {
804        // A float clock: an erased record before written ones writes no clock, which the
805        // table's reader refuses; the error names the record, from 1, and where it starts.
806        let layout = layout(&[("time (s)", FieldKind::F32), ("alt (m)", FieldKind::F32)]);
807        let mut bytes = Vec::new();
808        for (t, alt) in [(0.0_f32, 1.0_f32), (0.1, 2.0)] {
809            bytes.extend(t.to_le_bytes());
810            bytes.extend(alt.to_le_bytes());
811        }
812        bytes.extend([0xFF; 8]);
813        bytes.extend(0.3_f32.to_le_bytes());
814        bytes.extend(3.0_f32.to_le_bytes());
815        match read(&bytes, &layout) {
816            Err(LogError::Binary {
817                offset, message, ..
818            }) => {
819                assert_eq!(offset, 16);
820                assert!(message.starts_with("record 3: "), "{message}");
821            }
822            other => panic!("expected the record named, got {other:?}"),
823        }
824    }
825
826    #[test]
827    fn a_layout_survives_its_json() {
828        let mut layout = clock_and_pressure();
829        layout.byte_order = ByteOrder::Big;
830        let json = serde_json::to_string(&layout).unwrap();
831        assert_eq!(
832            json,
833            r#"{"fields":[{"name":"time__ms","kind":"u32"},{"name":"pressure__kpa","kind":"f32"}],"byte_order":"big"}"#
834        );
835        assert_eq!(serde_json::from_str::<RecordLayout>(&json).unwrap(), layout);
836        // The byte order defaults to little-endian.
837        let little: RecordLayout =
838            serde_json::from_str(r#"{"fields":[{"name":"t (s)","kind":"f64"}]}"#).unwrap();
839        assert_eq!(little.byte_order, ByteOrder::Little);
840    }
841}