hpr_flightdata/
import.rs

1//! One entry point for every format HPR Sim reads: give [`read`] a file's bytes, and it
2//! recognizes the layout from the content, never the file's name, and reads it into the record
3//! every reader fills ([`FlightLog`]).
4//!
5//! The layouts it recognizes, the GPS logs by their own marks, the rest by the first line that
6//! isn't blank:
7//!
8//! | Format | Recognized by |
9//! |---|---|
10//! | [KML](crate::kml) | its root element is `kml`, after any XML declaration and comments |
11//! | [NMEA 0183](crate::nmea) | at least 3 of its first 50 lines that aren't blank open with a sentence whose checksum checks out, such as `$GPGGA,…*hh` |
12//! | [Featherweight GPS CSV](crate::fwgps) | the first line opens `UTCTIME,UNIXTIME,` or `TRACKER,DATE,TIME,` and names the latitude and longitude |
13//! | [PerfectFlite `.pf2`](crate::perfectflite) | names PerfectFlite, such as `PerfectFlite Pnut`, in a file with no `**!!Data:` line (a `.pfa`'s) |
14//! | [PerfectFlite CSV and text](crate::perfectflite::read_export) | `Time`, `Altitude` and more of an export's names, by commas or tabs |
15//! | [AltOS CSV](crate::altos) | starts `#version,`, or opens with `#` comments before a `#version,` or `version,serial,` header |
16//! | [Raven FIP CSV](crate::raven) | opens with a time column and holds the Raven's `Axial Accel (Gs)` channel |
17//! | [Blue Raven CSV](crate::blueraven) | opens `Year,Month,Day,Time,Flight_Time_(s),Sync,` |
18//! | [RRC3 CSV](crate::rrc3) | `Time`, `Altitude`, `Pressure`, `Velocity`, then any of `Temperature`, `Events`, `Voltages` |
19//! | [Eggtimer CSV](crate::eggtimer) | `T,Alt,` and more of Eggtimer's column names |
20//!
21//! Text is read as UTF-8, or as UTF-16 after its byte-order mark ([`decode`]), with any byte-order
22//! mark dropped; a byte that isn't UTF-8 becomes U+FFFD, as a `.pf2`'s comments may be in another
23//! encoding and no reader uses them.
24//!
25//! ```
26//! use hpr_flightdata::{import, log::LogFormat};
27//!
28//! let text = "Time,Altitude,Velocity\n0.00,0,0\n0.05,1,8\n0.10,3,30\n";
29//! assert_eq!(import::detect(text.as_bytes()), Some(LogFormat::PerfectFliteCsv));
30//! let log = import::read(text.as_bytes())?;
31//! assert_eq!(log.channels.len(), 2); // the altitude and the velocity; the time is their clock
32//! # Ok::<(), hpr_flightdata::error::LogError>(())
33//! ```
34
35use std::borrow::Cow;
36
37use crate::error::LogError;
38use crate::log::{FlightLog, LogFormat};
39use crate::{altos, blueraven, eggtimer, fwgps, kml, nmea, perfectflite, raven, rrc3};
40
41/// The formats [`read`] recognizes, in the order it tries them.
42pub const FORMATS: &[LogFormat] = &[
43    LogFormat::Kml,
44    LogFormat::Nmea0183,
45    LogFormat::FeatherweightGpsCsv,
46    LogFormat::PerfectFlitePf2,
47    LogFormat::AltosCsv,
48    LogFormat::RavenFipCsv,
49    LogFormat::BlueRavenCsv,
50    LogFormat::Rrc3Csv,
51    LogFormat::EggtimerCsv,
52    LogFormat::PerfectFliteCsv,
53    LogFormat::PerfectFliteText,
54];
55
56/// A file's bytes as text: UTF-8, or UTF-16 if the file opens with UTF-16's byte-order mark, as
57/// Debrief's notes say some mDACS exports of an RRC3's flight are (none of the project's sample
58/// files is); any byte-order mark dropped, and a byte, or a pair of them, that isn't a character
59/// replaced by U+FFFD.
60pub fn decode(bytes: &[u8]) -> Cow<'_, str> {
61    let utf16 = |bytes: &[u8], unit: fn([u8; 2]) -> u16| {
62        let (pairs, odd) = bytes.as_chunks::<2>();
63        // An odd byte at the end is half a character.
64        let odd = (!odd.is_empty()).then_some(char::REPLACEMENT_CHARACTER);
65        Cow::Owned(
66            char::decode_utf16(pairs.iter().map(|pair| unit(*pair)))
67                .map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER))
68                .chain(odd)
69                .collect(),
70        )
71    };
72    if let Some(rest) = bytes.strip_prefix(b"\xff\xfe") {
73        utf16(rest, u16::from_le_bytes)
74    } else if let Some(rest) = bytes.strip_prefix(b"\xfe\xff") {
75        utf16(rest, u16::from_be_bytes)
76    } else {
77        let bytes = bytes.strip_prefix(b"\xef\xbb\xbf").unwrap_or(bytes);
78        String::from_utf8_lossy(bytes)
79    }
80}
81
82/// The format of a log, recognized from its first line that isn't blank; `None` for a layout HPR
83/// Sim doesn't read.
84pub fn detect(bytes: &[u8]) -> Option<LogFormat> {
85    detect_text(&decode(bytes))
86}
87
88fn detect_text(text: &str) -> Option<LogFormat> {
89    // Lines end in CR, LF or both, as the readers take them.
90    let mut lines = text
91        .split(['\r', '\n'])
92        .map(str::trim)
93        .filter(|line| !line.is_empty());
94    let first = lines.next()?;
95    // A GPS log is told by its own marks, which need not be on its first line: an NMEA file may
96    // open with the receiver's power-up noise, a KML file with an XML declaration.
97    if kml::recognizes(text) {
98        return Some(LogFormat::Kml);
99    }
100    if nmea::recognizes(text) {
101        return Some(LogFormat::Nmea0183);
102    }
103    if fwgps::recognizes(text) {
104        return Some(LogFormat::FeatherweightGpsCsv);
105    }
106    let altos_header =
107        |line: &str| line.starts_with("#version,") || line.starts_with("version,serial,");
108    if first.to_ascii_lowercase().contains("perfectflite") {
109        // A `.pfa` names PerfectFlite too, but writes its samples after a `**!!Data:` line, one
110        // value a line: a layout no reader knows yet (M7.1c).
111        let pfa = text
112            .split(['\r', '\n'])
113            .any(|line| line.trim_start().starts_with("**!!"));
114        (!pfa).then_some(LogFormat::PerfectFlitePf2)
115    } else if first.starts_with('#') && !altos_header(first) {
116        // Only AltOS's CSV opens with `#` comments, such as `# Comments` or `# product TeleMetrum`,
117        // before its header.
118        std::iter::once(first)
119            .chain(lines)
120            .find(|line| !line.starts_with('#') || altos_header(line))
121            .filter(|line| altos_header(line))
122            .map(|_| LogFormat::AltosCsv)
123    } else if altos_header(first) {
124        Some(LogFormat::AltosCsv)
125    } else if first.starts_with("Time") && first.contains("Axial Accel (Gs)") {
126        // The Raven's export opens with a time column; a Blue Raven's names the same channel
127        // further along a header that opens with the date.
128        Some(LogFormat::RavenFipCsv)
129    } else if blueraven::recognizes(text) {
130        Some(LogFormat::BlueRavenCsv)
131    } else if rrc3::recognizes(text) {
132        Some(LogFormat::Rrc3Csv)
133    } else if eggtimer::recognizes(text) {
134        Some(LogFormat::EggtimerCsv)
135    } else {
136        match perfectflite::export_separator(first)? {
137            ',' => Some(LogFormat::PerfectFliteCsv),
138            _ => Some(LogFormat::PerfectFliteText),
139        }
140    }
141}
142
143/// Reads a flight log in any format HPR Sim reads.
144///
145/// # Errors
146///
147/// [`LogError::Unrecognized`] for a layout HPR Sim doesn't read, naming those it does; otherwise
148/// whatever the format's reader refuses.
149pub fn read(bytes: &[u8]) -> Result<FlightLog, LogError> {
150    let text = decode(bytes);
151    match detect_text(&text) {
152        Some(LogFormat::PerfectFlitePf2) => perfectflite::read(&text),
153        Some(LogFormat::PerfectFliteCsv | LogFormat::PerfectFliteText) => {
154            perfectflite::read_export(&text)
155        }
156        Some(LogFormat::AltosCsv) => altos::read(&text),
157        Some(LogFormat::RavenFipCsv) => raven::read(&text),
158        Some(LogFormat::BlueRavenCsv) => blueraven::read(&text),
159        Some(LogFormat::Rrc3Csv) => rrc3::read(&text),
160        Some(LogFormat::EggtimerCsv) => eggtimer::read(&text),
161        Some(LogFormat::FeatherweightGpsCsv) => fwgps::read(&text),
162        Some(LogFormat::Kml) => kml::read(&text),
163        Some(LogFormat::Nmea0183) => nmea::read(&text),
164        None => {
165            let known: Vec<&str> = FORMATS.iter().map(|format| format.name()).collect();
166            Err(LogError::Unrecognized {
167                message: format!(
168                    "the file is in none of the layouts HPR Sim reads so far: {}",
169                    known.join(", ")
170                ),
171            })
172        }
173    }
174}
175
176#[cfg(test)]
177mod tests {
178    use super::*;
179
180    #[test]
181    fn each_format_is_recognized_by_its_marks() {
182        let cases = [
183            ("PerfectFlite Pnut\n", Some(LogFormat::PerfectFlitePf2)),
184            (
185                "\u{feff}\n\n#version,serial,flight,time\n",
186                Some(LogFormat::AltosCsv),
187            ),
188            (
189                "# Altus Metrum flight log\n# serial 1\nversion,serial,flight,time\n",
190                Some(LogFormat::AltosCsv),
191            ),
192            ("#Component=Raven\n", None),
193            (
194                "#foo\n#version,serial,flight,time\n",
195                Some(LogFormat::AltosCsv),
196            ),
197            ("# a note\nTime,Altitude,Velocity\n", None),
198            (
199                "PerfectFlite Alt15K/WD\nApogee: 900\n**!!Data:\n1200\n",
200                None,
201            ),
202            (
203                "Year,Month,Day,Time,Flight_Time_(s),Axial Accel (Gs)\n",
204                None,
205            ),
206            (
207                "Time,Altitude,Velocity\r0.00,0,0\r0.05,1,8\r",
208                Some(LogFormat::PerfectFliteCsv),
209            ),
210            (
211                "Time@Axial Accel (Gs),Axial Accel (Gs),bILBA\n",
212                Some(LogFormat::RavenFipCsv),
213            ),
214            (
215                "Time,LiftOff,Axial Accel (Gs),Baro (Atm)\n",
216                Some(LogFormat::RavenFipCsv),
217            ),
218            (
219                "Time, Altitude, Velocity\n",
220                Some(LogFormat::PerfectFliteCsv),
221            ),
222            (
223                "Time\tAltitude\tTemp.\tVoltage\n",
224                Some(LogFormat::PerfectFliteText),
225            ),
226            // A team's own export in feet: the generic reader's, M7.1c.
227            ("time_s,altitude_ft,temperature,voltage\n", None),
228            ("Time,Pressure\n", None),
229            ("time,altitude\n", None),
230            ("Time,Altitude\n", None),
231            (
232                "Time,Altitude,Pressure,Velocity,Temperature,Events,Voltages\n",
233                Some(LogFormat::Rrc3Csv),
234            ),
235            (
236                "T,Alt,Veloc,FAlt,FVeloc,LDA,LowV,Apogee,N-O,Drogue,Main\r\r\n",
237                Some(LogFormat::EggtimerCsv),
238            ),
239            (
240                "Year,Month,Day,Time,Flight_Time_(s),Sync,Gyro_X\n",
241                Some(LogFormat::BlueRavenCsv),
242            ),
243            (
244                "<?xml version=\"1.0\"?>\n<!-- a track -->\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"></kml>\n",
245                Some(LogFormat::Kml),
246            ),
247            ("<?xml version=\"1.0\"?>\n<gpx></gpx>\n", None),
248            (
249                "\u{fffd}\u{fffd}x\r\n\
250                 $GPGGA,120000.00,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*67\r\n\
251                 $GPRMC,120001.00,A,4807.038,N,01131.000,E,022.4,084.4,230394,003.1,W*4B\r\n\
252                 $GPGSA,A,3,04,05,,09,12,,,24,,,,,2.5,1.3,2.1*39\r\n",
253                Some(LogFormat::Nmea0183),
254            ),
255            // A sentence quoted inside another log's line isn't NMEA.
256            (
257                "t=1 rx $GPGGA,120000.00,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*67\n",
258                None,
259            ),
260            (
261                "UTCTIME,UNIXTIME,ALT,LAT,LON,#SATS,FIX,HORZV,VERTV,HEAD,FLAGS\n",
262                Some(LogFormat::FeatherweightGpsCsv),
263            ),
264            (
265                "TRACKER,DATE,TIME,LAT,LON,ALT,FIX,HORZV,VERTV,HEAD,FLAGS,#TOT\r\n",
266                Some(LogFormat::FeatherweightGpsCsv),
267            ),
268            ("TRACKER,DATE,TIME,RSSI\n", None),
269            ("", None),
270        ];
271        for (text, format) in cases {
272            assert_eq!(detect(text.as_bytes()), format, "{text:?}");
273        }
274    }
275
276    #[test]
277    fn utf16_is_read_by_its_byte_order_mark() {
278        let text = "Time\tAltitude é\n";
279        let mut le = vec![0xff, 0xfe];
280        let mut be = vec![0xfe, 0xff];
281        for unit in text.encode_utf16() {
282            le.extend(unit.to_le_bytes());
283            be.extend(unit.to_be_bytes());
284        }
285        assert_eq!(decode(&le), text);
286        assert_eq!(decode(&be), text);
287        // An odd byte at the end, and half a surrogate pair, are each one replaced character.
288        le.push(b'x');
289        assert_eq!(decode(&le), format!("{text}\u{fffd}"));
290        assert_eq!(decode(&[0xff, 0xfe, 0x3d, 0xd8]), "\u{fffd}");
291        assert_eq!(decode(b"\xef\xbb\xbfa\xff"), "a\u{fffd}");
292    }
293
294    #[test]
295    fn a_layout_no_reader_knows_is_refused_naming_those_that_exist() {
296        let error = read(b"Time,Pressure\n0,101325\n").unwrap_err();
297        let LogError::Unrecognized { message } = &error else {
298            panic!("refused for another reason: {error:?}");
299        };
300        for format in FORMATS {
301            assert!(message.contains(format.name()), "{message}");
302        }
303    }
304}
hpr_flightdata/import.rs

hpr_flightdata/
import.rs
