1use std::borrow::Cow;
40
41use schemars::JsonSchema;
42use serde::de::DeserializeOwned;
43use serde::{Deserialize, Serialize};
44
45use crate::capture;
46use crate::error::LogError;
47use crate::generic::{self, Mapping};
48use crate::import;
49use crate::log::FlightLog;
50use crate::records::{self, ByteOrder, Field, RecordLayout};
51use crate::{archive, raven, workbook, xls};
52
53pub const MAX_FILE_BYTES: usize = 1 << 20;
56
57#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
60#[serde(deny_unknown_fields)]
61pub struct Layout {
62 pub fields: Vec<Field>,
67 #[serde(default)]
69 pub byte_order: ByteOrder,
70 #[serde(default, skip_serializing_if = "Option::is_none")]
74 pub block: Option<String>,
75}
76
77impl Layout {
78 pub fn record_layout(&self) -> RecordLayout {
80 RecordLayout {
81 fields: self.fields.clone(),
82 byte_order: self.byte_order,
83 }
84 }
85}
86
87pub fn mapping_from_json(text: &str) -> Result<Mapping, LogError> {
94 from_json(text, "mapping").map_err(|message| LogError::Mapping { message })
95}
96
97pub fn layout_from_json(text: &str) -> Result<Layout, LogError> {
103 from_json(text, "layout").map_err(|message| LogError::Layout { message })
104}
105
106#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
112#[serde(deny_unknown_fields)]
113pub struct FipHeader {
114 pub header: String,
118 #[serde(default)]
121 #[schemars(inner(range(min = 1)))]
122 pub pasted: Vec<usize>,
123}
124
125pub fn fip_header_from_json(text: &str) -> Result<FipHeader, LogError> {
131 from_json(text, "FIP header").map_err(|message| LogError::FipHeader { message })
132}
133
134pub fn read_by_fip_header(
144 bytes: &[u8],
145 stated: &FipHeader,
146 sheet: Option<usize>,
147) -> Result<FlightLog, LogError> {
148 let text = match (workbook_sheets(bytes).transpose()?, sheet) {
149 (None, None) => import::decode(bytes).into_owned(),
150 (None, Some(_)) => {
151 return Err(LogError::Choice {
152 message: "the file isn't a spreadsheet's workbook, so it has no sheet to choose"
153 .to_owned(),
154 });
155 }
156 (Some(sheets), chosen) => {
157 let count = sheets.len();
158 let index = match chosen {
159 Some(index) => index,
160 None if count == 1 => 0,
161 None => {
162 return Err(LogError::Choice {
163 message: format!("the workbook has {count} sheets: choose one, from 0"),
164 });
165 }
166 };
167 sheets
168 .into_iter()
169 .nth(index)
170 .ok_or(LogError::Choice {
171 message: format!(
172 "the workbook has {count} {}, numbered from 0: it has no sheet {index}",
173 if count == 1 { "sheet" } else { "sheets" }
174 ),
175 })?
176 .text
177 }
178 };
179 raven::read_rows_around(&stated.header, &text, &stated.pasted).map_err(|error| match error {
182 LogError::Mapping { message } => LogError::FipHeader { message },
183 other => other,
184 })
185}
186
187#[must_use]
191pub fn json_text(bytes: &[u8]) -> Option<String> {
192 let utf16 = |rest: &[u8], unit: fn([u8; 2]) -> u16| {
193 let (pairs, odd) = rest.as_chunks::<2>();
194 if !odd.is_empty() {
195 return None;
196 }
197 char::decode_utf16(pairs.iter().map(|pair| unit(*pair)))
198 .collect::<Result<String, _>>()
199 .ok()
200 };
201 match bytes {
202 [0xEF, 0xBB, 0xBF, rest @ ..] => std::str::from_utf8(rest).ok().map(str::to_owned),
203 [0xFF, 0xFE, rest @ ..] => utf16(rest, u16::from_le_bytes),
204 [0xFE, 0xFF, rest @ ..] => utf16(rest, u16::from_be_bytes),
205 _ if bytes.contains(&0) => None,
208 _ => std::str::from_utf8(bytes).ok().map(str::to_owned),
209 }
210}
211
212fn from_json<T: DeserializeOwned>(text: &str, what: &str) -> Result<T, String> {
215 if text.len() > MAX_FILE_BYTES {
216 return Err(format!(
217 "a {what} file is at most {MAX_FILE_BYTES} bytes; this one is {}",
218 text.len()
219 ));
220 }
221 let mut json = serde_json::Deserializer::from_str(text);
222 let value = serde_path_to_error::deserialize(&mut json).map_err(|error| {
223 let words = error.inner().to_string();
224 let mut segments: Vec<_> = error.path().iter().collect();
225 if words.starts_with("unknown field") {
227 segments.pop();
228 }
229 let words = cut(&words, MAX_WORDS_CHARS);
231 match path(&segments) {
232 path if path.is_empty() => words,
233 path => format!("`{path}`: {words}"),
234 }
235 })?;
236 json.end().map_err(|error| error.to_string())?;
237 Ok(value)
238}
239
240const MAX_WORDS_CHARS: usize = 1000;
243
244fn cut(text: &str, chars: usize) -> String {
246 if text.chars().count() <= chars {
247 text.to_owned()
248 } else {
249 let mut cut: String = text.chars().take(chars - 1).collect();
250 cut.push('…');
251 cut
252 }
253}
254
255fn path(segments: &[&serde_path_to_error::Segment]) -> String {
257 use serde_path_to_error::Segment;
258 let mut text = String::new();
259 for segment in segments {
260 let key = match segment {
261 Segment::Seq { index } => {
262 text.push_str(&format!("[{index}]"));
263 continue;
264 }
265 Segment::Map { key } => key.as_str(),
266 Segment::Enum { variant } => variant.as_str(),
267 Segment::Unknown => "?",
268 };
269 if !text.is_empty() {
270 text.push('.');
271 }
272 text.push_str(key);
273 }
274 text
275}
276
277pub fn header_mapping(
288 bytes: &[u8],
289 sheet: Option<usize>,
290) -> Result<(Mapping, Vec<String>), LogError> {
291 generic::header_reading(&table_text(bytes, sheet)?)
292}
293
294pub fn read_by_mapping(
301 bytes: &[u8],
302 mapping: &Mapping,
303 sheet: Option<usize>,
304) -> Result<FlightLog, LogError> {
305 generic::read_with(&table_text(bytes, sheet)?, mapping)
306}
307
308pub fn read_by_layout(bytes: &[u8], layout: &Layout) -> Result<FlightLog, LogError> {
316 let text = import::decode(bytes);
317 let is_capture = capture::recognizes(&text);
318 match (&layout.block, is_capture) {
319 (Some(block), true) => capture::read(&text, block, &layout.record_layout()),
320 (Some(block), false) => Err(LogError::Layout {
321 message: format!(
322 "the layout names the block {block:?} of a serial console's capture, and the \
323 file isn't one. For a file of the records' bytes, leave `block` out"
324 ),
325 }),
326 (None, true) => {
327 let names: Vec<String> = capture::blocks(&text)?
328 .into_iter()
329 .map(|block| format!("{:?}", block.name))
330 .collect();
331 Err(LogError::Layout {
332 message: format!(
333 "the file is a serial console's capture of records as base64, its blocks {}. \
334 The layout must name the block its records are in, as `block`",
335 names.join(", ")
336 ),
337 })
338 }
339 (None, false) => records::read(bytes, &layout.record_layout()),
340 }
341}
342
343fn workbook_sheets(bytes: &[u8]) -> Option<Result<Vec<workbook::Sheet>, LogError>> {
345 if archive::is_archive(bytes) {
346 Some(workbook::sheets(bytes))
347 } else if xls::recognizes(bytes) {
348 Some(xls::sheets(bytes))
349 } else {
350 None
351 }
352}
353
354fn table_text(bytes: &[u8], sheet: Option<usize>) -> Result<Cow<'_, str>, LogError> {
357 let refused = |message: String| LogError::Mapping { message };
358 match (workbook_sheets(bytes), sheet) {
359 (Some(Ok(sheets)), Some(sheet)) => {
360 let count = sheets.len();
361 sheets
362 .into_iter()
363 .nth(sheet)
364 .map(|found| Cow::Owned(found.text))
365 .ok_or_else(|| {
366 refused(format!(
367 "the workbook has no sheet {sheet}: {}",
368 match count {
369 0 => "it has none".to_owned(),
370 1 => "its one sheet is 0".to_owned(),
371 n => format!("its {n} sheets are 0 to {}", n - 1),
372 }
373 ))
374 })
375 }
376 (Some(Err(error)), Some(_)) => Err(refused(format!(
377 "a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook that reads: {error}"
378 ))),
379 (None, Some(_)) => Err(refused(
380 "a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook".to_owned(),
381 )),
382 (Some(Ok(mut sheets)), None) if sheets.len() == 1 => {
383 Ok(Cow::Owned(sheets.swap_remove(0).text))
384 }
385 (Some(Ok(sheets)), None) => {
386 let names = workbook::listed(
387 sheets.iter().map(|found| found.name.as_str()).enumerate(),
388 |index, name| format!("{index} {name:?}"),
389 );
390 Err(refused(format!(
391 "the file is a spreadsheet's workbook, its sheets {}: name the sheet the table is \
392 on, from 0",
393 if names.is_empty() { "none" } else { &names }
394 )))
395 }
396 (Some(Err(LogError::NotThisFormat { .. })) | None, None) => Ok(import::decode(bytes)),
399 (Some(Err(error)), None) => Err(error),
400 }
401}
402
403#[cfg(test)]
404mod tests {
405 use super::*;
406 use crate::log::Quantity;
407
408 const TABLE: &str = "\
411t (ms),alt (ft),accel (g)
4120,0,1.0
41350,1.5,6.2
414100,6.0,6.1
415150,13.4,0.4
416";
417
418 fn mapping(primary: &str) -> String {
419 format!(
420 r#"{{
421 "header_line": 1,
422 "separator": ",",
423 "decimal_comma": false,
424 "clock": 0,
425 "primary": {primary},
426 "columns": [
427 {{"name": "t (ms)", "quantity": null, "unit": "ms"}},
428 {{"name": "alt (ft)", "quantity": "altitude_above_ground", "unit": "ft"}},
429 {{"name": "accel (g)", "quantity": "acceleration", "unit": "g"}}
430 ]
431}}"#
432 )
433 }
434
435 fn message(error: LogError) -> String {
436 match error {
437 LogError::Mapping { message } | LogError::Layout { message } => message,
438 other => panic!("not a mapping's or layout's refusal: {other}"),
439 }
440 }
441
442 #[test]
443 fn a_marked_primary_track_is_the_records_altitude_on_its_clock() {
444 let mapping = mapping_from_json(&mapping("1")).unwrap();
445 let log = read_by_mapping(TABLE.as_bytes(), &mapping, None).unwrap();
446 assert_eq!(log.time_s, [0.0, 0.05, 0.1, 0.15]);
447 let feet = [0.0, 1.5, 6.0, 13.4].map(|ft: f64| ft * 0.3048);
448 assert_eq!(log.altitude_m, feet);
449 let altitude: Vec<_> = log.channels_of(Quantity::AltitudeAboveGround).collect();
451 assert_eq!(altitude.len(), 1);
452 assert_eq!(altitude[0].values, feet);
453 assert_eq!(log.channels.len(), 2);
454 }
455
456 #[test]
457 fn a_mapping_that_marks_no_primary_track_leaves_it_empty() {
458 for primary in ["null", "none"] {
459 let text = if primary == "none" {
460 mapping("null").replace(" \"primary\": null,\n", "")
461 } else {
462 mapping(primary)
463 };
464 let mapping = mapping_from_json(&text).unwrap();
465 assert_eq!(mapping.primary, None);
466 let log = read_by_mapping(TABLE.as_bytes(), &mapping, None).unwrap();
467 assert!(log.time_s.is_empty() && log.altitude_m.is_empty());
468 assert_eq!(log.channels.len(), 2);
469 }
470 }
471
472 #[test]
473 fn a_primary_track_that_isnt_an_altitude_above_the_pad_is_refused() {
474 let refused = |primary: &str, table: &str| {
475 let mapping = mapping_from_json(&mapping(primary)).unwrap();
476 message(read_by_mapping(table.as_bytes(), &mapping, None).unwrap_err())
477 };
478 assert_eq!(refused("3", TABLE), "the primary track is column 3, of 3");
479 assert_eq!(
480 refused("0", TABLE),
481 "the primary track, \"t (ms)\", is the clock's column"
482 );
483 assert_eq!(
484 refused("2", TABLE),
485 "the primary track, \"accel (g)\", is mapped to `acceleration`: it must be \
486 `altitude_above_ground`, the barometric altitude above the pad"
487 );
488 let skipped = mapping("1").replace(r#""unit": "ft"}"#, r#""unit": "ft", "skip": true}"#);
489 let mapping_skipped = mapping_from_json(&skipped).unwrap();
490 assert_eq!(
491 message(read_by_mapping(TABLE.as_bytes(), &mapping_skipped, None).unwrap_err()),
492 "the primary track, \"alt (ft)\", is a column the mapping leaves out"
493 );
494 let kept = mapping("1").replace(
495 r#""quantity": "altitude_above_ground", "unit": "ft""#,
496 r#""quantity": null, "unit": null"#,
497 );
498 let mapping_kept = mapping_from_json(&kept).unwrap();
499 assert_eq!(
500 message(read_by_mapping(TABLE.as_bytes(), &mapping_kept, None).unwrap_err()),
501 "the primary track, \"alt (ft)\", is kept as written: it must be \
502 `altitude_above_ground`, the barometric altitude above the pad"
503 );
504 let impossible = TABLE.replace("100,6.0,6.1", "100,999999999,6.1");
506 assert_eq!(
507 refused("1", &impossible),
508 "the primary track, \"alt (ft)\", holds values that contradict its unit, \"ft\""
509 );
510 }
511
512 #[test]
513 fn a_primary_track_needs_altitudes_on_a_clock_that_goes_forward() {
514 let mapping = mapping_from_json(&mapping("1")).unwrap();
515 let empty = "t (ms),alt (ft),accel (g)\n0,,1.0\n50,,6.2\n100,,6.1\n";
516 assert_eq!(
517 message(read_by_mapping(empty.as_bytes(), &mapping, None).unwrap_err()),
518 "the primary track, \"alt (ft)\", holds no number"
519 );
520 for (back, line) in [("100,6.0", 4), ("50,6.0", 4)] {
523 let table = TABLE.replace("150,13.4", back);
524 let error = message(read_by_mapping(table.as_bytes(), &mapping, None).unwrap_err());
525 assert!(
526 error.starts_with(&format!(
527 "the primary track, \"alt (ft)\", needs a clock that goes forward, and the \
528 clock, \"t (ms)\", doesn't on line {}",
529 line + 1
530 )),
531 "{error}"
532 );
533 }
534 let unmarked = mapping_from_json(&super::tests::mapping("null")).unwrap();
536 let table = TABLE.replace("150,13.4", "50,6.0");
537 let log = read_by_mapping(table.as_bytes(), &unmarked, None).unwrap();
538 assert!(log.notes.iter().any(|n| n.contains("steps back 1 times")));
539 }
540
541 #[test]
542 fn a_mapping_file_is_refused_naming_the_field_it_fails() {
543 let refused = |text: &str| message(mapping_from_json(text).unwrap_err());
544 let misspelled = mapping("1").replace("\"separator\"", "\"seperator\"");
545 assert!(
546 refused(&misspelled).starts_with("unknown field `seperator`, expected one of"),
547 "{}",
548 refused(&misspelled)
549 );
550 let missing = mapping("1").replace(" \"clock\": 0,\n", "");
551 assert!(refused(&missing).starts_with("missing field `clock`"));
552 let wrong = mapping("1").replace(r#""unit": "ft""#, r#""unit": 3"#);
553 assert!(
554 refused(&wrong).starts_with("`columns[1].unit`: invalid type: integer `3`"),
555 "{}",
556 refused(&wrong)
557 );
558 assert!(
559 refused(&wrong).ends_with("line 9 column 71"),
560 "{}",
561 refused(&wrong)
562 );
563 let column_key = mapping("1").replace(r#""unit": "ft""#, r#""unit": "ft", "units": "ft""#);
564 assert!(refused(&column_key).starts_with("`columns[1]`: unknown field `units`"));
565 let quantity = mapping("1").replace("acceleration", "accel");
566 assert!(
567 refused(&quantity).starts_with("`columns[2].quantity`: unknown variant `accel`"),
568 "{}",
569 refused(&quantity)
570 );
571 assert!(refused(&mapping("-1")).starts_with("`primary`: invalid value: integer `-1`"));
572 assert!(refused(&format!("{} {{}}", mapping("1"))).starts_with("trailing characters"));
573 let long = format!("{}{}", mapping("1"), " ".repeat(MAX_FILE_BYTES));
574 assert!(refused(&long).starts_with("a mapping file is at most 1048576 bytes"));
575 let huge = mapping("1").replace("acceleration", &"x".repeat(100_000));
577 let said = refused(&huge);
578 assert!(
579 said.starts_with("`columns[2].quantity`: unknown variant `xxx"),
580 "{said}"
581 );
582 assert!(said.ends_with('…'), "{said}");
583 assert_eq!(
584 said.chars().count(),
585 "`columns[2].quantity`: ".chars().count() + MAX_WORDS_CHARS
586 );
587 }
588
589 #[test]
590 fn a_files_text_is_utf8_or_utf16_with_its_mark() {
591 let text = mapping("1");
592 assert_eq!(json_text(text.as_bytes()), Some(text.clone()));
593 let marked = [&[0xEF, 0xBB, 0xBF][..], text.as_bytes()].concat();
594 assert_eq!(json_text(&marked), Some(text.clone()));
595 let little: Vec<u8> = [0xFF, 0xFE]
596 .into_iter()
597 .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
598 .collect();
599 assert_eq!(json_text(&little), Some(text.clone()));
600 let big: Vec<u8> = [0xFE, 0xFF]
601 .into_iter()
602 .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
603 .collect();
604 assert_eq!(json_text(&big), Some(text.clone()));
605 assert_eq!(
607 mapping_from_json(&json_text(&little).unwrap()).unwrap(),
608 mapping_from_json(&text).unwrap()
609 );
610 assert_eq!(json_text(&little[..little.len() - 1]), None);
611 assert_eq!(json_text(&little[2..]), None, "UTF-16 with no mark");
612 assert_eq!(json_text(&big[2..]), None, "UTF-16 with no mark");
613 assert_eq!(json_text(&[0xFF, 0xFE, 0x00, 0xD8]), None);
614 assert_eq!(json_text(b"{\xff}"), None);
615 }
616
617 const LAYOUT: &str = r#"{
618 "fields": [
619 {"name": "time__ms", "kind": "u32"},
620 {"name": "pressure__pa", "kind": "f32"}
621 ]
622}"#;
623
624 fn records() -> Vec<u8> {
626 let mut bytes = Vec::new();
627 for (time, pressure) in [(0_u32, 101_325.0_f32), (20, 101_300.0), (40, 101_250.0)] {
628 bytes.extend(time.to_le_bytes());
629 bytes.extend(pressure.to_le_bytes());
630 }
631 bytes
632 }
633
634 #[test]
635 fn a_layout_file_reads_records_as_the_library_does() {
636 let layout = layout_from_json(LAYOUT).unwrap();
637 assert_eq!(layout.byte_order, ByteOrder::Little);
638 assert_eq!(layout.block, None);
639 let log = read_by_layout(&records(), &layout).unwrap();
640 assert_eq!(
641 log,
642 records::read(&records(), &layout.record_layout()).unwrap()
643 );
644 assert_eq!(log.channels[0].values, [101_325.0, 101_300.0, 101_250.0]);
645 }
646
647 #[test]
648 fn a_layout_reads_a_captures_block_it_names_and_refuses_one_it_doesnt() {
649 let encoded = base64(&records());
650 let capture = format!(
651 "uart:~$ log dump\n----++++//[[( data start )]]\\\\++++----\n{encoded}\n\
652 ----++++//[[( data end )]]\\\\++++----\nuart:~$\n"
653 );
654 assert!(capture::recognizes(&capture), "{capture}");
655 let named = LAYOUT.replace("\n ]\n}", "\n ],\n \"block\": \"data\"\n}");
656 let layout = layout_from_json(&named).unwrap();
657 let log = read_by_layout(capture.as_bytes(), &layout).unwrap();
658 assert_eq!(
659 log,
660 capture::read(&capture, "data", &layout.record_layout()).unwrap()
661 );
662 let unnamed = layout_from_json(LAYOUT).unwrap();
663 assert_eq!(
664 message(read_by_layout(capture.as_bytes(), &unnamed).unwrap_err()),
665 "the file is a serial console's capture of records as base64, its blocks \"data\". \
666 The layout must name the block its records are in, as `block`"
667 );
668 assert!(
669 message(read_by_layout(&records(), &layout).unwrap_err())
670 .starts_with("the layout names the block \"data\" of a serial console's capture")
671 );
672 }
673
674 #[test]
675 fn a_layout_file_is_refused_naming_the_field_it_fails() {
676 let refused = |text: &str| message(layout_from_json(text).unwrap_err());
677 let kind = LAYOUT.replace("\"f32\"", "\"float\"");
678 assert!(
679 refused(&kind).starts_with("`fields[1].kind`: unknown variant `float`"),
680 "{}",
681 refused(&kind)
682 );
683 let key = LAYOUT.replace("\"fields\"", "\"field\"");
684 assert!(refused(&key).starts_with("unknown field `field`"));
685 let order = LAYOUT.replace("\n ]\n}", "\n ],\n \"byte_order\": \"middle\"\n}");
686 assert!(refused(&order).starts_with("`byte_order`: unknown variant `middle`"));
687 }
688
689 #[test]
690 fn a_sheet_named_for_a_text_file_is_refused() {
691 let mapping = mapping_from_json(&mapping("1")).unwrap();
692 let error = message(read_by_mapping(TABLE.as_bytes(), &mapping, Some(0)).unwrap_err());
693 assert!(
694 error.starts_with("a sheet is named, and the file isn't an `.xlsx` or `.xls` workbook"),
695 "{error}"
696 );
697 }
698
699 fn base64(bytes: &[u8]) -> String {
701 const ALPHABET: &[u8; 64] =
702 b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
703 let mut text = String::new();
704 for chunk in bytes.chunks(3) {
705 let word = chunk.iter().enumerate().fold(0_u32, |word, (i, byte)| {
706 word | u32::from(*byte) << (16 - 8 * i)
707 });
708 for i in 0..4 {
709 if i <= chunk.len() {
710 text.push(char::from(ALPHABET[(word >> (18 - 6 * i) & 63) as usize]));
711 } else {
712 text.push('=');
713 }
714 }
715 }
716 text
717 }
718}