1use std::borrow::Cow;
219
220use schemars::JsonSchema;
221use serde::{Deserialize, Serialize};
222
223use crate::error::LogError;
224use crate::log::{Channel, Event, FlightLog, LogFormat, Quantity, Stated};
225use crate::table::{self, numbered_lines, short};
226
227const FORMAT: &str = "table read by its header";
229
230pub const MAX_COLUMNS: usize = 512;
233
234pub const MAX_EVENTS_PER_COLUMN: usize = 64;
237
238pub const MAX_MARKS: usize = 64;
241
242const HEAD_LINES: usize = 64;
244
245const SEPARATORS: [char; 4] = [',', ';', '\t', '|'];
247
248const NOT_THE_QUANTITY: [&str; 16] = [
251 "accuracy",
252 "error",
253 "err",
254 "std",
255 "stdev",
256 "stddev",
257 "sigma",
258 "var",
259 "variance",
260 "uncertainty",
261 "diff",
262 "differential",
263 "dynamic",
264 "dyn",
265 "pitot",
266 "delta",
267];
268
269const AMBIGUOUS_UNITS: [&str; 8] = ["g", "gs", "mg", "c", "f", "v", "k", "a"];
273
274const SINGLE_LETTER_SUFFIXES: [&str; 6] = ["s", "m", "g", "c", "f", "v"];
277
278#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
282#[serde(deny_unknown_fields)]
283pub struct Mapping {
284 pub header_line: Option<usize>,
287 pub separator: char,
289 pub decimal_comma: bool,
292 pub clock: usize,
294 #[serde(default, skip_serializing_if = "Option::is_none")]
299 pub primary: Option<usize>,
300 pub columns: Vec<ColumnMapping>,
302 #[serde(default, skip_serializing_if = "Option::is_none")]
306 pub clock_wrap: Option<f64>,
307 #[serde(default, skip_serializing_if = "RowForm::is_cells")]
309 pub rows: RowForm,
310}
311
312#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
314#[serde(rename_all = "snake_case")]
315pub enum RowForm {
316 #[default]
318 Cells,
319 List,
321 Stamped,
325}
326
327impl RowForm {
328 fn is_cells(&self) -> bool {
329 *self == Self::Cells
330 }
331}
332
333#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
335#[serde(deny_unknown_fields)]
336pub struct ColumnMapping {
337 pub name: String,
339 pub quantity: Option<Quantity>,
342 pub unit: Option<String>,
346 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
349 pub skip: bool,
350}
351
352#[derive(Debug, Clone, Copy, PartialEq, Eq)]
354enum Dimension {
355 Time,
356 Length,
357 Speed,
358 Acceleration,
359 Angle,
360 AngularRate,
361 Temperature,
362 Voltage,
363 Current,
364 Pressure,
365}
366
367impl Dimension {
368 fn word(self) -> &'static str {
369 match self {
370 Self::Time => "a time",
371 Self::Length => "a length",
372 Self::Speed => "a speed",
373 Self::Acceleration => "an acceleration",
374 Self::Angle => "an angle",
375 Self::AngularRate => "a rate of turn",
376 Self::Temperature => "a temperature",
377 Self::Voltage => "a voltage",
378 Self::Current => "a current",
379 Self::Pressure => "a pressure",
380 }
381 }
382
383 fn bounds(self) -> Option<(f64, f64)> {
386 match self {
387 Self::Pressure => Some((0.0, MAX_PRESSURE_PA)),
388 Self::Temperature => Some((
389 CELSIUS_ZERO_K + MIN_TEMPERATURE_C,
390 CELSIUS_ZERO_K + MAX_TEMPERATURE_C,
391 )),
392 Self::Length => Some((-MAX_LENGTH_M, MAX_LENGTH_M)),
393 Self::Speed => Some((-MAX_SPEED_M_S, MAX_SPEED_M_S)),
394 Self::Acceleration => {
395 Some((-MAX_ACCELERATION_G * G0_M_S2, MAX_ACCELERATION_G * G0_M_S2))
396 }
397 Self::AngularRate => Some((-MAX_RATE_DEG_S, MAX_RATE_DEG_S)),
398 Self::Voltage | Self::Current => Some((-MAX_ELECTRIC, MAX_ELECTRIC)),
399 Self::Time | Self::Angle => None,
400 }
401 }
402}
403
404const MAX_PRESSURE_PA: f64 = 110_000.0;
407const MIN_TEMPERATURE_C: f64 = -100.0;
409const MAX_TEMPERATURE_C: f64 = 150.0;
410const MAX_LENGTH_M: f64 = 150_000.0;
412const MAX_SPEED_M_S: f64 = 3_000.0;
414const MAX_ACCELERATION_G: f64 = 1_000.0;
416const MAX_RATE_DEG_S: f64 = 50_000.0;
418const MAX_ELECTRIC: f64 = 100.0;
420const MIN_CLOCK_STEP_S: f64 = 1e-4;
422const MAX_CLOCK_STEP_S: f64 = 60.0;
423
424const CELSIUS_ZERO_K: f64 = 273.15;
426const FOOT_M: f64 = 0.3048;
428const G0_M_S2: f64 = 9.806_65;
430const RADIAN_DEG: f64 = 180.0 / std::f64::consts::PI;
432
433#[derive(Debug, Clone, Copy, PartialEq)]
436struct Unit {
437 dimension: Dimension,
438 multiply: f64,
439 divide: f64,
440 add: f64,
441}
442
443impl Unit {
444 const fn new(dimension: Dimension, multiply: f64, divide: f64) -> Self {
445 Self {
446 dimension,
447 multiply,
448 divide,
449 add: 0.0,
450 }
451 }
452
453 const fn offset(dimension: Dimension, multiply: f64, divide: f64, add: f64) -> Self {
454 Self {
455 dimension,
456 multiply,
457 divide,
458 add,
459 }
460 }
461
462 fn convert(self, written: f64) -> f64 {
463 written * self.multiply / self.divide + self.add
464 }
465}
466
467const FAHRENHEIT: Unit = Unit::offset(
469 Dimension::Temperature,
470 5.0,
471 9.0,
472 CELSIUS_ZERO_K - 32.0 * 5.0 / 9.0,
473);
474const CELSIUS: Unit = Unit::offset(Dimension::Temperature, 1.0, 1.0, CELSIUS_ZERO_K);
475
476pub fn unit_spellings() -> impl Iterator<Item = &'static str> {
479 UNIT_TABLE.iter().map(|(name, _)| *name)
480}
481
482const UNIT_TABLE: &[(&str, Unit)] = &[
484 ("s", Unit::new(Dimension::Time, 1.0, 1.0)),
485 ("sec", Unit::new(Dimension::Time, 1.0, 1.0)),
486 ("secs", Unit::new(Dimension::Time, 1.0, 1.0)),
487 ("second", Unit::new(Dimension::Time, 1.0, 1.0)),
488 ("seconds", Unit::new(Dimension::Time, 1.0, 1.0)),
489 ("ms", Unit::new(Dimension::Time, 1.0, 1e3)),
490 ("msec", Unit::new(Dimension::Time, 1.0, 1e3)),
491 ("millis", Unit::new(Dimension::Time, 1.0, 1e3)),
492 ("millisecond", Unit::new(Dimension::Time, 1.0, 1e3)),
493 ("milliseconds", Unit::new(Dimension::Time, 1.0, 1e3)),
494 ("us", Unit::new(Dimension::Time, 1.0, 1e6)),
495 ("µs", Unit::new(Dimension::Time, 1.0, 1e6)),
496 ("μs", Unit::new(Dimension::Time, 1.0, 1e6)),
497 ("usec", Unit::new(Dimension::Time, 1.0, 1e6)),
498 ("microsecond", Unit::new(Dimension::Time, 1.0, 1e6)),
499 ("microseconds", Unit::new(Dimension::Time, 1.0, 1e6)),
500 ("m", Unit::new(Dimension::Length, 1.0, 1.0)),
501 ("meter", Unit::new(Dimension::Length, 1.0, 1.0)),
502 ("meters", Unit::new(Dimension::Length, 1.0, 1.0)),
503 ("ft", Unit::new(Dimension::Length, FOOT_M, 1.0)),
504 ("feet", Unit::new(Dimension::Length, FOOT_M, 1.0)),
505 ("foot", Unit::new(Dimension::Length, FOOT_M, 1.0)),
506 ("km", Unit::new(Dimension::Length, 1e3, 1.0)),
507 ("m/s", Unit::new(Dimension::Speed, 1.0, 1.0)),
508 ("mps", Unit::new(Dimension::Speed, 1.0, 1.0)),
509 ("m_s", Unit::new(Dimension::Speed, 1.0, 1.0)),
510 ("ft/s", Unit::new(Dimension::Speed, FOOT_M, 1.0)),
511 ("fps", Unit::new(Dimension::Speed, FOOT_M, 1.0)),
512 ("ft_s", Unit::new(Dimension::Speed, FOOT_M, 1.0)),
513 ("km/h", Unit::new(Dimension::Speed, 1.0, 3.6)),
514 ("kmh", Unit::new(Dimension::Speed, 1.0, 3.6)),
515 ("kph", Unit::new(Dimension::Speed, 1.0, 3.6)),
516 ("m_per_s", Unit::new(Dimension::Speed, 1.0, 1.0)),
517 ("meters_per_s", Unit::new(Dimension::Speed, 1.0, 1.0)),
518 ("ft_per_s", Unit::new(Dimension::Speed, FOOT_M, 1.0)),
519 ("feet_per_s", Unit::new(Dimension::Speed, FOOT_M, 1.0)),
520 ("mph", Unit::new(Dimension::Speed, 0.447_04, 1.0)),
521 ("kn", Unit::new(Dimension::Speed, 1_852.0, 3_600.0)),
522 ("kt", Unit::new(Dimension::Speed, 1_852.0, 3_600.0)),
523 ("knots", Unit::new(Dimension::Speed, 1_852.0, 3_600.0)),
524 ("m/s2", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
525 ("m/s²", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
526 ("m/s^2", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
527 ("m/s/s", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
528 ("m_s2", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
529 ("mps2", Unit::new(Dimension::Acceleration, 1.0, 1.0)),
530 ("ft/s2", Unit::new(Dimension::Acceleration, FOOT_M, 1.0)),
531 ("ft/s²", Unit::new(Dimension::Acceleration, FOOT_M, 1.0)),
532 ("ft/s^2", Unit::new(Dimension::Acceleration, FOOT_M, 1.0)),
533 ("g", Unit::new(Dimension::Acceleration, G0_M_S2, 1.0)),
534 ("gs", Unit::new(Dimension::Acceleration, G0_M_S2, 1.0)),
535 ("mg", Unit::new(Dimension::Acceleration, G0_M_S2, 1e3)),
536 ("milli-g", Unit::new(Dimension::Acceleration, G0_M_S2, 1e3)),
537 ("deg", Unit::new(Dimension::Angle, 1.0, 1.0)),
538 ("degs", Unit::new(Dimension::Angle, 1.0, 1.0)),
539 ("degree", Unit::new(Dimension::Angle, 1.0, 1.0)),
540 ("degrees", Unit::new(Dimension::Angle, 1.0, 1.0)),
541 ("°", Unit::new(Dimension::Angle, 1.0, 1.0)),
542 ("rad", Unit::new(Dimension::Angle, RADIAN_DEG, 1.0)),
543 ("radians", Unit::new(Dimension::Angle, RADIAN_DEG, 1.0)),
544 ("deg/s", Unit::new(Dimension::AngularRate, 1.0, 1.0)),
545 ("°/s", Unit::new(Dimension::AngularRate, 1.0, 1.0)),
546 ("dps", Unit::new(Dimension::AngularRate, 1.0, 1.0)),
547 ("deg_per_s", Unit::new(Dimension::AngularRate, 1.0, 1.0)),
548 ("degrees/s", Unit::new(Dimension::AngularRate, 1.0, 1.0)),
549 ("rad/s", Unit::new(Dimension::AngularRate, RADIAN_DEG, 1.0)),
550 ("rad_s", Unit::new(Dimension::AngularRate, RADIAN_DEG, 1.0)),
551 (
552 "rad_per_s",
553 Unit::new(Dimension::AngularRate, RADIAN_DEG, 1.0),
554 ),
555 (
556 "radians/s",
557 Unit::new(Dimension::AngularRate, RADIAN_DEG, 1.0),
558 ),
559 ("degc", CELSIUS),
560 ("°c", CELSIUS),
561 ("c", CELSIUS),
562 ("celsius", CELSIUS),
563 ("degf", FAHRENHEIT),
564 ("°f", FAHRENHEIT),
565 ("f", FAHRENHEIT),
566 ("fahrenheit", FAHRENHEIT),
567 ("k", Unit::new(Dimension::Temperature, 1.0, 1.0)),
568 ("kelvin", Unit::new(Dimension::Temperature, 1.0, 1.0)),
569 ("v", Unit::new(Dimension::Voltage, 1.0, 1.0)),
570 ("volt", Unit::new(Dimension::Voltage, 1.0, 1.0)),
571 ("volts", Unit::new(Dimension::Voltage, 1.0, 1.0)),
572 ("mv", Unit::new(Dimension::Voltage, 1.0, 1e3)),
573 ("millivolts", Unit::new(Dimension::Voltage, 1.0, 1e3)),
574 ("a", Unit::new(Dimension::Current, 1.0, 1.0)),
575 ("amp", Unit::new(Dimension::Current, 1.0, 1.0)),
576 ("amps", Unit::new(Dimension::Current, 1.0, 1.0)),
577 ("ma", Unit::new(Dimension::Current, 1.0, 1e3)),
578 ("milliamps", Unit::new(Dimension::Current, 1.0, 1e3)),
579 ("pa", Unit::new(Dimension::Pressure, 1.0, 1.0)),
580 ("pascal", Unit::new(Dimension::Pressure, 1.0, 1.0)),
581 ("pascals", Unit::new(Dimension::Pressure, 1.0, 1.0)),
582 ("hpa", Unit::new(Dimension::Pressure, 100.0, 1.0)),
583 ("mbar", Unit::new(Dimension::Pressure, 100.0, 1.0)),
584 ("millibar", Unit::new(Dimension::Pressure, 100.0, 1.0)),
585 ("kpa", Unit::new(Dimension::Pressure, 1e3, 1.0)),
586 ("bar", Unit::new(Dimension::Pressure, 1e5, 1.0)),
587 ("atm", Unit::new(Dimension::Pressure, 101_325.0, 1.0)),
588 ("psi", Unit::new(Dimension::Pressure, 6_894.757, 1.0)),
589 ("inhg", Unit::new(Dimension::Pressure, 3_386.389, 1.0)),
590];
591
592fn key(written: &str) -> String {
594 written
595 .chars()
596 .filter(|c| !c.is_whitespace())
597 .flat_map(char::to_lowercase)
598 .collect()
599}
600
601fn unit_of(written: &str) -> Option<Unit> {
603 let key = key(written);
604 UNIT_TABLE
605 .iter()
606 .find(|(name, _)| *name == key)
607 .map(|(_, unit)| *unit)
608}
609
610fn split_name(written: &str) -> (String, Option<String>) {
613 let written = written.trim();
614 if let Some(close) = written.chars().last().filter(|c| *c == ')' || *c == ']') {
615 let open = if close == ')' { '(' } else { '[' };
616 if let Some(at) = written.rfind(open) {
617 let inner = &written[at + open.len_utf8()..written.len() - close.len_utf8()];
618 let base = written[..at].trim_end_matches([' ', '_']);
619 if unit_of(inner).is_some() {
620 return (base.to_owned(), Some(key(inner)));
621 }
622 let mut words = inner.split_whitespace();
623 if let Some(first) = words.next()
624 && unit_of(first).is_some()
625 {
626 let qualifier: Vec<&str> = words.collect();
627 let name = if qualifier.is_empty() {
628 base.to_owned()
629 } else {
630 format!("{base} {}", qualifier.join(" "))
631 };
632 return (name, Some(key(first)));
633 }
634 return (written.to_owned(), None);
635 }
636 }
637 let words: Vec<&str> = written.split('_').filter(|w| !w.is_empty()).collect();
638 for count in (1..=4).rev() {
639 if words.len() <= count {
640 continue;
641 }
642 let ending = words[words.len() - count..].join("_");
643 let ending_key = key(&ending);
644 if ending_key.chars().count() == 1 && !SINGLE_LETTER_SUFFIXES.contains(&ending_key.as_str())
645 {
646 continue;
647 }
648 if unit_of(&ending).is_some() {
649 return (words[..words.len() - count].join("_"), Some(ending_key));
650 }
651 }
652 (written.to_owned(), None)
653}
654
655fn words(name: &str) -> Vec<String> {
657 name.split(|c: char| !c.is_alphanumeric())
658 .filter(|w| !w.is_empty())
659 .map(str::to_lowercase)
660 .collect()
661}
662
663fn is_time_name(words: &[String]) -> bool {
665 words
666 .iter()
667 .any(|w| matches!(w.as_str(), "t" | "ts" | "uptime" | "timestamp") || w.starts_with("time"))
668}
669
670fn hints(words: &[String]) -> Vec<Dimension> {
672 let mut hints = Vec::new();
673 let mut hint = |dimension: Dimension, said: bool| {
674 if said && !hints.contains(&dimension) {
675 hints.push(dimension);
676 }
677 };
678 let any = |test: &dyn Fn(&str) -> bool| words.iter().any(|w| test(w));
679 hint(
680 Dimension::Length,
681 any(&|w| {
682 w.starts_with("alt")
683 || w.ends_with("alt")
684 || matches!(w, "height" | "elev" | "elevation" | "agl" | "msl" | "asl")
685 }),
686 );
687 hint(
688 Dimension::Pressure,
689 any(&|w| w.starts_with("press") || w.starts_with("baro") || w == "pression"),
690 );
691 hint(Dimension::Temperature, any(&|w| w.starts_with("temp")));
692 hint(
693 Dimension::Voltage,
694 any(&|w| w.starts_with("volt") || w.starts_with("batt") || w.starts_with("vbat")),
695 );
696 hint(
697 Dimension::Current,
698 any(&|w| w.starts_with("curr") || w.starts_with("amp")),
699 );
700 hint(
701 Dimension::Acceleration,
702 any(&|w| w.starts_with("acc") || w.ends_with("acc") || w.contains("accel")),
703 );
704 hint(
705 Dimension::AngularRate,
706 any(&|w| w.starts_with("gyro") || w == "rate"),
707 );
708 hint(
709 Dimension::Speed,
710 any(&|w| {
711 w.starts_with("vel") || w.starts_with("speed") || matches!(w, "vx" | "vy" | "vz" | "v")
712 }),
713 );
714 hint(
715 Dimension::Angle,
716 any(&|w| {
717 matches!(
718 w,
719 "lat"
720 | "latitude"
721 | "lon"
722 | "lng"
723 | "long"
724 | "longitude"
725 | "heading"
726 | "angle"
727 | "tilt"
728 | "roll"
729 | "pitch"
730 | "yaw"
731 | "course"
732 | "bearing"
733 | "cog"
734 )
735 }),
736 );
737 hint(Dimension::Time, is_time_name(words));
738 hints
739}
740
741fn quantity_of(dimension: Dimension, words: &[String]) -> Quantity {
744 let says = |options: &[&str]| words.iter().any(|w| options.contains(&w.as_str()));
745 let mentions = |part: &str| words.iter().any(|w| w.contains(part));
746 match dimension {
747 Dimension::Length => {
748 let agl = says(&["agl"]);
749 if (mentions("gps") || mentions("gnss")) && !agl {
750 Quantity::GnssAltitude
751 } else if mentions("baro") && agl {
752 Quantity::AltitudeAboveGround
753 } else if mentions("baro") && (says(&["msl", "asl"])) {
754 Quantity::AltitudeMsl
755 } else if mentions("inertial") {
756 Quantity::InertialAltitude
757 } else {
758 Quantity::Altitude
759 }
760 }
761 Dimension::Time => Quantity::Duration,
762 Dimension::Angle => {
763 if says(&["lat", "latitude"]) {
764 Quantity::Latitude
765 } else if says(&["lon", "lng", "long", "longitude"]) {
766 Quantity::Longitude
767 } else {
768 Quantity::Angle
769 }
770 }
771 Dimension::Speed => Quantity::Speed,
772 Dimension::Acceleration => Quantity::Acceleration,
773 Dimension::AngularRate => Quantity::AngularRate,
774 Dimension::Temperature => Quantity::Temperature,
775 Dimension::Voltage => Quantity::Voltage,
776 Dimension::Current => Quantity::Current,
777 Dimension::Pressure => Quantity::Pressure,
778 }
779}
780
781fn dimension_of(quantity: Quantity) -> Option<Dimension> {
783 match quantity {
784 Quantity::AltitudeAboveGround
785 | Quantity::AltitudeMsl
786 | Quantity::GnssAltitude
787 | Quantity::InertialAltitude
788 | Quantity::Altitude
789 | Quantity::Distance => Some(Dimension::Length),
790 Quantity::Pressure => Some(Dimension::Pressure),
791 Quantity::Speed => Some(Dimension::Speed),
792 Quantity::Acceleration => Some(Dimension::Acceleration),
793 Quantity::Temperature => Some(Dimension::Temperature),
794 Quantity::Voltage => Some(Dimension::Voltage),
795 Quantity::Current => Some(Dimension::Current),
796 Quantity::Latitude | Quantity::Longitude | Quantity::Angle => Some(Dimension::Angle),
797 Quantity::AngularRate => Some(Dimension::AngularRate),
798 Quantity::TimeOfDay | Quantity::Duration => Some(Dimension::Time),
799 Quantity::Number | Quantity::Unrecognized => None,
800 }
801}
802
803fn bounds_of(quantity: Quantity, dimension: Dimension) -> Option<(f64, f64)> {
805 match quantity {
806 Quantity::Latitude => Some((-90.0, 90.0)),
807 Quantity::Longitude => Some((-180.0, 180.0)),
808 _ => dimension.bounds(),
809 }
810}
811
812fn infer(written: &str) -> (ColumnMapping, Option<String>) {
814 let (name, unit_key) = split_name(written);
815 let words = words(&name);
816 let kept = |why: String| {
817 (
818 ColumnMapping {
819 name: written.to_owned(),
820 quantity: None,
821 unit: None,
822 skip: false,
823 },
824 Some(why),
825 )
826 };
827 let Some((unit_key, unit)) = unit_key.and_then(|k| unit_of(&k).map(|u| (k, u))) else {
828 return kept(format!(
829 "the column {} states no unit the reader knows: kept as written",
830 quoted(written)
831 ));
832 };
833 if let Some(word) = words
834 .iter()
835 .find(|w| NOT_THE_QUANTITY.contains(&w.as_str()))
836 {
837 return kept(format!(
838 "the column {}'s name says {word:?}, an estimate's error or a difference, not the \
839 quantity itself: kept as written",
840 quoted(written)
841 ));
842 }
843 let hints = hints(&words);
844 if hints.is_empty() && matches!(unit.dimension, Dimension::Length | Dimension::Time) {
845 return kept(format!(
846 "the column {} is {} whose name says no altitude or time: kept as written",
847 quoted(written),
848 unit.dimension.word()
849 ));
850 }
851 if hints.is_empty() && AMBIGUOUS_UNITS.contains(&unit_key.as_str()) {
852 return kept(format!(
853 "the column {}'s unit, {unit_key}, may stand for something other than {}, and its \
854 name doesn't say which: kept as written",
855 quoted(written),
856 unit.dimension.word()
857 ));
858 }
859 if unit.dimension == Dimension::Angle && hints.contains(&Dimension::AngularRate) {
860 return kept(format!(
861 "the column {}'s name says a rate of turn and its unit, {unit_key}, an angle: kept as \
862 written",
863 quoted(written)
864 ));
865 }
866 if !hints.is_empty() && !hints.contains(&unit.dimension) {
867 let named: Vec<&str> = hints.iter().map(|d| d.word()).collect();
868 return kept(format!(
869 "the column {}'s name says {} and its unit, {unit_key}, {}: kept as written",
870 quoted(written),
871 named.join(" or "),
872 unit.dimension.word()
873 ));
874 }
875 (
876 ColumnMapping {
877 name: written.to_owned(),
878 quantity: Some(quantity_of(unit.dimension, &words)),
879 unit: Some(unit_key),
880 skip: false,
881 },
882 None,
883 )
884}
885
886fn quoted(name: &str) -> String {
887 format!("{:?}", short(name))
888}
889
890fn cells(line: &str, separator: char, limit: usize) -> Vec<Cow<'_, str>> {
893 if !line.contains('"') {
894 return line
895 .split(separator)
896 .take(limit)
897 .map(|c| Cow::Borrowed(c.trim()))
898 .collect();
899 }
900 let mut cells = Vec::new();
901 let mut cell = String::new();
902 let mut quoted = false;
903 let mut chars = line.chars().peekable();
904 while let Some(c) = chars.next() {
905 if quoted {
906 if c == '"' {
907 if chars.peek() == Some(&'"') {
908 cell.push('"');
909 chars.next();
910 } else {
911 quoted = false;
912 }
913 } else {
914 cell.push(c);
915 }
916 } else if c == '"' && cell.trim().is_empty() {
917 cell.clear();
918 quoted = true;
919 } else if c == separator {
920 cells.push(Cow::Owned(cell.trim().to_owned()));
921 if cells.len() == limit {
922 return cells;
923 }
924 cell.clear();
925 } else {
926 cell.push(c);
927 }
928 }
929 cells.push(Cow::Owned(cell.trim().to_owned()));
930 cells
931}
932
933fn number(cell: &str, decimal_comma: bool) -> Option<f64> {
935 table::number_in_mark(cell, decimal_comma)
936}
937
938fn decimal_comma(text: &str, header_line: usize, separator: char) -> bool {
942 if separator == ',' {
943 return false;
944 }
945 table::decimal_comma_shown(
946 numbered_lines(text)
947 .filter(|(n, l)| *n > header_line && !skipped(l))
948 .flat_map(|(_, line)| cells(line, separator, MAX_COLUMNS + 1)),
949 )
950}
951
952fn skipped(line: &str) -> bool {
954 let line = line.trim();
955 line.is_empty() || line.starts_with('#')
956}
957
958fn sep_line(line: &str) -> Option<char> {
960 let rest = line.trim().strip_prefix("sep=")?;
961 let mut chars = rest.chars();
962 let separator = chars
963 .next()
964 .filter(|c| SEPARATORS.contains(c) || *c == ' ')?;
965 chars.next().is_none().then_some(separator)
966}
967
968fn find_header(text: &str) -> Option<(Mapping, Vec<String>)> {
971 let mut named: Option<char> = None;
972 let mut lines = numbered_lines(text)
973 .take(HEAD_LINES)
974 .filter(|(_, l)| !skipped(l));
975 let (line, header) = loop {
976 let (number, line) = lines.next()?;
977 if named.is_none()
978 && let Some(separator) = sep_line(line)
979 {
980 named = Some(separator);
981 continue;
982 }
983 break (number, line);
984 };
985 let separator = named.unwrap_or_else(|| {
986 let mut best = (SEPARATORS[0], 0);
987 for separator in SEPARATORS {
988 let count = cells(header, separator, MAX_COLUMNS + 1).len();
989 if count > best.1 {
990 best = (separator, count);
991 }
992 }
993 best.0
994 });
995 let names = cells(header, separator, MAX_COLUMNS + 1);
996 if names.len() < 2 || names.len() > MAX_COLUMNS {
997 return None;
998 }
999 if names
1000 .iter()
1001 .any(|name| table::number(name, separator != ',').is_some())
1002 {
1003 return None;
1004 }
1005 let mut columns = Vec::with_capacity(names.len());
1006 let mut notes = Vec::new();
1007 let mut clock = None;
1008 for name in &names {
1009 let (mapping, why) = infer(name);
1010 let is_clock = clock.is_none()
1011 && mapping
1012 .unit
1013 .as_deref()
1014 .and_then(unit_of)
1015 .map(|u| u.dimension)
1016 == Some(Dimension::Time)
1017 && is_time_name(&words(&split_name(name).0));
1018 if is_clock {
1019 clock = Some(columns.len());
1020 } else if let Some(why) = why {
1021 notes.push(why);
1022 }
1023 columns.push(mapping);
1024 }
1025 let clock = clock?;
1026 let (_, first) = numbered_lines(text)
1027 .skip_while(|(number, _)| *number <= line)
1028 .find(|(_, l)| !skipped(l))?;
1029 let decimal_comma = decimal_comma(text, line, separator);
1030 let first = cells(first, separator, MAX_COLUMNS + 1);
1031 number(first.get(clock)?, decimal_comma)?;
1032 if let Some(column) = columns.get_mut(clock) {
1033 column.quantity = None;
1034 }
1035 Some((
1036 Mapping {
1037 header_line: Some(line),
1038 separator,
1039 decimal_comma,
1040 clock,
1041 primary: None,
1042 columns,
1043 clock_wrap: None,
1044 rows: RowForm::Cells,
1045 },
1046 notes,
1047 ))
1048}
1049
1050pub fn recognizes(text: &str) -> bool {
1052 find_header(text).is_some()
1053}
1054
1055pub fn mapping(text: &str) -> Result<Mapping, LogError> {
1062 find_header(text)
1063 .map(|(mapping, _)| mapping)
1064 .ok_or_else(not_this_format)
1065}
1066
1067pub fn header_reading(text: &str) -> Result<(Mapping, Vec<String>), LogError> {
1074 find_header(text).ok_or_else(not_this_format)
1075}
1076
1077fn not_this_format() -> LogError {
1078 LogError::NotThisFormat {
1079 format: FORMAT,
1080 message: format!(
1081 "within its first {HEAD_LINES} lines there is no header naming a clock with a unit of \
1082 time, followed by a row with a number there. For example, `Time (s)` or \
1083 `time__us` names such a clock"
1084 ),
1085 }
1086}
1087
1088pub fn read(text: &str) -> Result<FlightLog, LogError> {
1094 let (mapping, notes) = find_header(text).ok_or_else(not_this_format)?;
1095 read_mapped(text, &mapping, notes)
1096}
1097
1098pub fn read_with(text: &str, mapping: &Mapping) -> Result<FlightLog, LogError> {
1112 read_mapped(text, mapping, Vec::new())
1113}
1114
1115fn mapping_error(message: String) -> LogError {
1116 LogError::Mapping { message }
1117}
1118
1119fn units(mapping: &Mapping) -> Result<(Unit, Vec<Option<Unit>>), LogError> {
1121 let width = mapping.columns.len();
1122 if width == 0 || width > MAX_COLUMNS {
1123 return Err(mapping_error(format!(
1124 "a mapping names from 1 to {MAX_COLUMNS} columns; this one names {width}"
1125 )));
1126 }
1127 if !SEPARATORS.contains(&mapping.separator) && mapping.separator != ' ' {
1128 return Err(mapping_error(format!(
1129 "the separator {:?} isn't one of `,`, `;`, a tab, `|` or a space",
1130 mapping.separator
1131 )));
1132 }
1133 if mapping.decimal_comma && mapping.separator == ',' {
1134 return Err(mapping_error(
1135 "a decimal comma needs a separator other than a comma".to_owned(),
1136 ));
1137 }
1138 let Some(clock) = mapping.columns.get(mapping.clock) else {
1139 return Err(mapping_error(format!(
1140 "the clock is column {}, of {width}",
1141 mapping.clock
1142 )));
1143 };
1144 if clock.skip {
1145 return Err(mapping_error(format!(
1146 "the clock, {}, is a column the mapping leaves out",
1147 quoted(&clock.name)
1148 )));
1149 }
1150 if let Some(turn) = mapping.clock_wrap
1151 && !(turn.is_finite() && turn > 0.0)
1152 {
1153 return Err(mapping_error(format!(
1154 "the clock's turn, {turn}, isn't a count above 0"
1155 )));
1156 }
1157 let clock_unit = clock
1158 .unit
1159 .as_deref()
1160 .and_then(unit_of)
1161 .filter(|u| u.dimension == Dimension::Time)
1162 .ok_or_else(|| {
1163 mapping_error(format!(
1164 "the clock, {}, has no unit of time the reader knows",
1165 quoted(&clock.name)
1166 ))
1167 })?;
1168 if mapping.rows == RowForm::Stamped
1169 && (mapping.clock != 0 || clock_unit.convert(1.0) != 1.0 || mapping.clock_wrap.is_some())
1170 {
1171 return Err(mapping_error(
1172 "stamped rows' clock is their stamp: column 0, in `s`, with no turn".to_owned(),
1173 ));
1174 }
1175 if let Some(primary) = mapping.primary {
1176 let Some(column) = mapping.columns.get(primary) else {
1177 return Err(mapping_error(format!(
1178 "the primary track is column {primary}, of {width}"
1179 )));
1180 };
1181 let refused = |why: &str| {
1182 mapping_error(format!(
1183 "the primary track, {}, {why}",
1184 quoted(&column.name)
1185 ))
1186 };
1187 if primary == mapping.clock {
1188 return Err(refused("is the clock's column"));
1189 }
1190 if column.skip {
1191 return Err(refused("is a column the mapping leaves out"));
1192 }
1193 if column.quantity != Some(Quantity::AltitudeAboveGround) {
1194 let mapped = column
1195 .quantity
1196 .and_then(|q| serde_json::to_value(q).ok())
1197 .and_then(|v| v.as_str().map(|name| format!("mapped to `{name}`")))
1198 .unwrap_or_else(|| "kept as written".to_owned());
1199 return Err(refused(&format!(
1200 "is {mapped}: it must be `altitude_above_ground`, the barometric altitude above \
1201 the pad"
1202 )));
1203 }
1204 }
1205 let mut units = Vec::with_capacity(width);
1206 for (index, column) in mapping.columns.iter().enumerate() {
1207 if index == mapping.clock || column.skip {
1208 units.push(None);
1209 continue;
1210 }
1211 let wanted = column.quantity.and_then(dimension_of);
1212 let unit = match (&column.unit, wanted) {
1213 (None, None) => None,
1214 (Some(written), Some(dimension)) => match unit_of(written) {
1215 Some(unit) if unit.dimension == dimension => Some(unit),
1216 Some(unit) => {
1217 return Err(mapping_error(format!(
1218 "the column {} is mapped to {} in {}, which measures {}",
1219 quoted(&column.name),
1220 dimension.word(),
1221 quoted(written),
1222 unit.dimension.word()
1223 )));
1224 }
1225 None => {
1226 return Err(mapping_error(format!(
1227 "the column {}'s unit, {}, isn't one the reader knows",
1228 quoted(&column.name),
1229 quoted(written)
1230 )));
1231 }
1232 },
1233 (None, Some(dimension)) => {
1234 return Err(mapping_error(format!(
1235 "the column {} is mapped to {} but states no unit",
1236 quoted(&column.name),
1237 dimension.word()
1238 )));
1239 }
1240 (Some(written), None) => {
1241 return Err(mapping_error(format!(
1242 "the column {} states a unit, {}, for a quantity that has none",
1243 quoted(&column.name),
1244 quoted(written)
1245 )));
1246 }
1247 };
1248 units.push(unit);
1249 }
1250 Ok((clock_unit, units))
1251}
1252
1253struct Rows {
1255 lines: Vec<usize>,
1256 clock: Vec<f64>,
1257 values: Vec<Vec<f64>>,
1258 text: Vec<(usize, Option<String>)>,
1260 changes: Vec<Vec<(usize, String)>>,
1263 last_word: Vec<(Option<String>, usize)>,
1265 marks: Vec<(usize, String)>,
1268 marks_left_out: usize,
1269 marks_after_last: usize,
1270}
1271
1272fn read_rows(text: &str, mapping: &Mapping) -> Result<Rows, LogError> {
1273 let width = mapping.columns.len();
1274 let syntax = |line: usize, message: String| LogError::Syntax {
1275 format: FORMAT,
1276 line,
1277 message,
1278 };
1279 let mut rows = Rows {
1280 lines: Vec::new(),
1281 clock: Vec::new(),
1282 values: vec![Vec::new(); width],
1283 text: vec![(0, None); width],
1284 changes: vec![Vec::new(); width],
1285 last_word: vec![(None, 0); width],
1286 marks: Vec::new(),
1287 marks_left_out: 0,
1288 marks_after_last: 0,
1289 };
1290 let mut pending_marks = Vec::new();
1292 let mut pending_past_cap = 0;
1293 let event_columns: Vec<bool> = mapping
1294 .columns
1295 .iter()
1296 .map(|c| is_event_name(&c.name))
1297 .collect();
1298 let start = mapping.header_line.unwrap_or(0);
1299 let mut header_skipped = false;
1300 for (line_number, line) in numbered_lines(text) {
1301 if line_number <= start || skipped(line) {
1302 continue;
1303 }
1304 if !header_skipped && mapping.header_line.is_none() && sep_line(line).is_some() {
1305 header_skipped = true;
1306 continue;
1307 }
1308 header_skipped = true;
1309 let (stamp, body) = match mapping.rows {
1310 RowForm::Cells => (None, line),
1311 RowForm::List => {
1312 let Some(body) = line
1313 .trim()
1314 .strip_prefix('[')
1315 .and_then(|b| b.strip_suffix(']'))
1316 else {
1317 return Err(syntax(
1318 line_number,
1319 "the row isn't a list between `[` and `]`".to_owned(),
1320 ));
1321 };
1322 (None, body)
1323 }
1324 RowForm::Stamped => match stamped(line) {
1325 Stamp::Row(seconds, body) => (Some(seconds), body),
1326 Stamp::None => {
1327 if rows.marks.len() + pending_marks.len() < MAX_MARKS {
1328 pending_marks.push(mark(line));
1329 } else {
1330 pending_past_cap += 1;
1331 }
1332 continue;
1333 }
1334 Stamp::Invalid => {
1335 return Err(syntax(
1336 line_number,
1337 "the row's stamp isn't a date and time, `YYYY-MM-DD HH:MM:SS`, followed \
1338 by `:`"
1339 .to_owned(),
1340 ));
1341 }
1342 },
1343 };
1344 let mut cells = cells(body, mapping.separator, width + 1);
1345 if stamp.is_some() {
1346 cells.insert(0, Cow::Borrowed(""));
1347 }
1348 if cells.len() != width {
1349 return Err(syntax(
1350 line_number,
1351 format!(
1352 "the row has {} cells, where the {} names {width}",
1353 if cells.len() > width {
1354 "more".to_owned()
1355 } else {
1356 cells.len().to_string()
1357 },
1358 if mapping.header_line.is_some() {
1359 "header"
1360 } else {
1361 "mapping"
1362 }
1363 ),
1364 ));
1365 }
1366 let clock_cell = cells.get(mapping.clock).map_or("", |c| c.as_ref());
1367 let Some(time) = stamp.or_else(|| number(clock_cell, mapping.decimal_comma)) else {
1368 return Err(syntax(
1369 line_number,
1370 format!("the clock's cell {} isn't a number", quoted(clock_cell)),
1371 ));
1372 };
1373 let row = rows.clock.len();
1374 rows.marks
1375 .extend(pending_marks.drain(..).map(|words| (row, words)));
1376 rows.marks_left_out += std::mem::take(&mut pending_past_cap);
1377 for (index, cell) in cells.iter().enumerate() {
1378 let value = number(cell, mapping.decimal_comma);
1379 if value.is_none() && !cell.is_empty() {
1380 let (count, first) = &mut rows.text[index];
1381 *count += 1;
1382 if first.is_none() {
1383 *first = Some(short(cell));
1384 }
1385 if event_columns[index] {
1386 let word = short(cell);
1387 let (last, left_out) = &mut rows.last_word[index];
1388 if last.as_ref() != Some(&word) {
1389 let changes = &mut rows.changes[index];
1390 if changes.len() < MAX_EVENTS_PER_COLUMN {
1391 changes.push((row, word.clone()));
1392 } else {
1393 *left_out += 1;
1394 }
1395 *last = Some(word);
1396 }
1397 }
1398 }
1399 rows.values[index].push(value.unwrap_or(f64::NAN));
1400 }
1401 rows.lines.push(line_number);
1402 rows.clock.push(time);
1403 }
1404 rows.marks_after_last = pending_marks.len() + pending_past_cap;
1405 Ok(rows)
1406}
1407
1408enum Stamp<'a> {
1410 Row(f64, &'a str),
1413 Invalid,
1416 None,
1418}
1419
1420fn stamped(line: &str) -> Stamp<'_> {
1421 let line = line.trim_start();
1422 let bytes = line.as_bytes();
1423 let shape = b"dddd-dd-dd dd:dd:dd";
1424 let shaped = bytes.len() >= shape.len()
1425 && shape.iter().zip(bytes).all(|(want, got)| {
1426 if *want == b'd' {
1427 got.is_ascii_digit()
1428 } else {
1429 want == got
1430 }
1431 });
1432 if !shaped {
1433 return Stamp::None;
1434 }
1435 let field = |from: usize, count: usize| -> i64 {
1437 line[from..from + count]
1438 .bytes()
1439 .fold(0, |n, b| n * 10 + i64::from(b - b'0'))
1440 };
1441 let (year, month, day) = (field(0, 4), field(5, 2), field(8, 2));
1442 let (hour, minute, second) = (field(11, 2), field(14, 2), field(17, 2));
1443 if !(1..=12).contains(&month)
1444 || !(1..=days_in_month(year, month)).contains(&day)
1445 || hour > 23
1446 || minute > 59
1447 || second > 59
1448 {
1449 return Stamp::Invalid;
1450 }
1451 let mut end = shape.len();
1452 let mut fraction = 0.0;
1453 if bytes.get(end) == Some(&b'.') {
1454 let count = bytes[end + 1..]
1455 .iter()
1456 .take_while(|b| b.is_ascii_digit())
1457 .count();
1458 if count == 0 || count > 9 {
1459 return Stamp::Invalid;
1460 }
1461 #[allow(
1462 clippy::cast_possible_truncation,
1463 reason = "1 to 9 digits, by the check above"
1464 )]
1465 let places = count as i32;
1466 fraction = field(end + 1, count) as f64 / 10f64.powi(places);
1467 end += 1 + count;
1468 }
1469 let Some(rest) = line[end..].strip_prefix(':') else {
1470 return Stamp::Invalid;
1471 };
1472 let rest = rest.trim();
1473 let rest = rest
1474 .strip_prefix('(')
1475 .and_then(|r| r.strip_suffix(')'))
1476 .map_or(rest, str::trim);
1477 let whole = days_from_civil(year, month, day) * 86_400 + hour * 3_600 + minute * 60 + second;
1478 Stamp::Row(whole as f64 + fraction, rest)
1479}
1480
1481fn mark(line: &str) -> String {
1483 short(
1484 line.trim()
1485 .trim_matches(|c: char| c == '=' || c == '-' || c == '*' || c.is_whitespace()),
1486 )
1487}
1488
1489fn days_in_month(year: i64, month: i64) -> i64 {
1491 match month {
1492 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29,
1493 2 => 28,
1494 4 | 6 | 9 | 11 => 30,
1495 _ => 31,
1496 }
1497}
1498
1499fn days_from_civil(year: i64, month: i64, day: i64) -> i64 {
1502 let year = if month <= 2 { year - 1 } else { year };
1503 let era = year.div_euclid(400);
1504 let year_of_era = year - era * 400;
1505 let shifted_month = (month + 9) % 12;
1506 let day_of_year = (153 * shifted_month + 2) / 5 + day - 1;
1507 let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
1508 era * 146_097 + day_of_era - 719_468
1509}
1510
1511fn counted(count: usize, one: &str, many: &str) -> String {
1513 format!("{count} {}", if count == 1 { one } else { many })
1514}
1515
1516fn unwrap_clock(clock: &mut [f64], turn: f64) -> (usize, usize) {
1520 let (mut past, mut turns) = (0, 0);
1521 let mut offset = 0.0;
1522 let mut previous: Option<f64> = None;
1523 for value in clock.iter_mut() {
1524 let counted = value.rem_euclid(turn);
1525 if counted != *value {
1526 past += 1;
1527 }
1528 if previous.is_some_and(|p| p - counted > turn / 2.0) {
1529 offset += turn;
1530 turns += 1;
1531 }
1532 previous = Some(counted);
1533 *value = counted + offset;
1534 }
1535 (past, turns)
1536}
1537
1538fn median_step(clock: &[f64]) -> Option<f64> {
1540 let mut steps: Vec<f64> = clock
1541 .windows(2)
1542 .map(|pair| pair[1] - pair[0])
1543 .filter(|step| *step > 0.0)
1544 .collect();
1545 if steps.is_empty() {
1546 return None;
1547 }
1548 steps.sort_by(f64::total_cmp);
1549 steps.get(steps.len() / 2).copied()
1550}
1551
1552fn is_event_name(name: &str) -> bool {
1555 words(name).iter().any(|w| {
1556 ["state", "phase", "status", "event", "mode"]
1557 .iter()
1558 .any(|word| w.ends_with(word) || w.strip_suffix('s') == Some(word))
1559 })
1560}
1561
1562fn read_mapped(
1563 text: &str,
1564 mapping: &Mapping,
1565 header_notes: Vec<String>,
1566) -> Result<FlightLog, LogError> {
1567 let (clock_unit, units) = units(mapping)?;
1568 let mut rows = read_rows(text, mapping)?;
1569 if rows.clock.is_empty() {
1570 return Err(LogError::NoData { format: FORMAT });
1571 }
1572 let turned = mapping
1573 .clock_wrap
1574 .map(|turn| (turn, unwrap_clock(&mut rows.clock, turn)));
1575 let clock_name = mapping
1576 .columns
1577 .get(mapping.clock)
1578 .map_or("", |c| c.name.as_str());
1579 let first_line = rows.lines.first().copied().unwrap_or(0);
1580 let unit_line = mapping.header_line.unwrap_or(first_line);
1581 let Some(step) = median_step(&rows.clock) else {
1582 return Err(LogError::Unit {
1583 format: FORMAT,
1584 line: unit_line,
1585 message: format!(
1586 "the clock, {}, never steps forward, so its values can't bear out its unit",
1587 quoted(clock_name)
1588 ),
1589 });
1590 };
1591 let step_s = clock_unit.convert(step);
1592 if !(MIN_CLOCK_STEP_S..=MAX_CLOCK_STEP_S).contains(&step_s) {
1593 return Err(LogError::Unit {
1594 format: FORMAT,
1595 line: unit_line,
1596 message: format!(
1597 "the clock, {}, steps by {step} at the median, {step_s} s in its unit, outside \
1598 {MIN_CLOCK_STEP_S} s to {MAX_CLOCK_STEP_S} s: its values contradict the unit",
1599 quoted(clock_name)
1600 ),
1601 });
1602 }
1603 let mut time_s = Vec::with_capacity(rows.clock.len());
1604 for (value, line) in rows.clock.iter().zip(&rows.lines) {
1605 time_s.push(table::finite(
1606 clock_unit.convert(*value),
1607 FORMAT,
1608 *line,
1609 clock_name,
1610 )?);
1611 }
1612 let mut notes = header_notes;
1615 if let Some((turn, (past, turns))) = turned {
1616 notes.push(format!(
1617 "the clock, {}, counts modulo {turn}: {} written past it taken modulo it, and it \
1618 turns over {}, each turn added",
1619 quoted(clock_name),
1620 counted(past, "value", "values"),
1621 match turns {
1622 1 => "once".to_owned(),
1623 n => format!("{n} times"),
1624 }
1625 ));
1626 }
1627 let skipped_columns = mapping.columns.iter().filter(|c| c.skip).count();
1628 if skipped_columns > 0 {
1629 notes.push(format!(
1630 "the mapping leaves out {skipped_columns} of the {} columns",
1631 mapping.columns.len()
1632 ));
1633 }
1634 if !rows.marks.is_empty() || rows.marks_left_out > 0 {
1635 notes.push(format!(
1636 "{} with no stamp: each a mark, an event at the next row's time{}",
1637 counted(rows.marks.len() + rows.marks_left_out, "line", "lines"),
1638 if rows.marks_left_out > 0 {
1639 format!(", the first {MAX_MARKS} kept")
1640 } else {
1641 String::new()
1642 }
1643 ));
1644 }
1645 if rows.marks_after_last > 0 {
1646 notes.push(format!(
1647 "{} with no stamp after the last row: left out",
1648 counted(rows.marks_after_last, "line", "lines")
1649 ));
1650 }
1651 let back = rows.clock.windows(2).filter(|p| p[1] < p[0]).count();
1652 if back > 0 {
1653 notes.push(format!(
1654 "the clock, {}, steps back {back} times; the rows are kept in the file's order",
1655 quoted(clock_name)
1656 ));
1657 }
1658 let mut channels = Vec::new();
1659 let mut primary_channel = None;
1660 let mut changes: Vec<(usize, &str)> = Vec::new();
1661 for (index, column) in mapping.columns.iter().enumerate() {
1662 if index == mapping.clock || column.skip {
1663 continue;
1664 }
1665 let values = &rows.values[index];
1666 let (text, first) = &rows.text[index];
1667 let first = quoted(first.as_deref().unwrap_or_default());
1668 if values.iter().all(|v| v.is_nan()) {
1669 if *text == 0 {
1670 notes.push(format!(
1671 "the column {} holds no value: left out",
1672 quoted(&column.name)
1673 ));
1674 } else if is_event_name(&column.name) {
1675 changes.extend(rows.changes[index].iter().map(|(r, w)| (*r, w.as_str())));
1676 let left_out = rows.last_word[index].1;
1677 if left_out > 0 {
1678 notes.push(format!(
1679 "the column {} changes its word {left_out} more times than the \
1680 {MAX_EVENTS_PER_COLUMN} kept as events",
1681 quoted(&column.name)
1682 ));
1683 }
1684 } else {
1685 notes.push(format!(
1686 "the column {} holds text, not numbers, such as {first}: left out",
1687 quoted(&column.name)
1688 ));
1689 }
1690 continue;
1691 }
1692 if *text > 0 {
1693 notes.push(format!(
1694 "the column {} holds {text} cells that aren't numbers, such as {first}: gaps",
1695 quoted(&column.name)
1696 ));
1697 }
1698 let quantity = column.quantity.unwrap_or(Quantity::Unrecognized);
1699 let (quantity, values) = match units[index] {
1700 Some(unit) => {
1701 let mut converted = Vec::with_capacity(values.len());
1702 for (value, line) in values.iter().zip(&rows.lines) {
1703 converted.push(table::finite(
1704 unit.convert(*value),
1705 FORMAT,
1706 *line,
1707 &column.name,
1708 )?);
1709 }
1710 let outside = bounds_of(quantity, unit.dimension).and_then(|(low, high)| {
1711 converted
1712 .iter()
1713 .zip(&rows.lines)
1714 .zip(values)
1715 .find(|((v, _), _)| **v < low || **v > high)
1716 .map(|((_, line), written)| (*line, *written, low, high))
1717 });
1718 match outside {
1719 Some((line, written, low, high)) => {
1720 notes.push(format!(
1721 "the column {}'s values contradict its unit, {}: {written} on line \
1722 {line} lies outside {low} to {high} {} once converted; kept as \
1723 written",
1724 quoted(&column.name),
1725 column.unit.as_deref().unwrap_or_default(),
1726 quantity.unit()
1727 ));
1728 (Quantity::Unrecognized, values.clone())
1729 }
1730 None => (quantity, converted),
1731 }
1732 }
1733 None => (quantity, values.clone()),
1734 };
1735 if mapping.primary == Some(index) {
1736 primary_channel = Some(channels.len());
1737 }
1738 channels.push(Channel {
1739 column: column.name.clone(),
1740 quantity,
1741 time_s: time_s.clone(),
1742 values,
1743 });
1744 }
1745 let (track_time_s, altitude_m) = primary_track(
1746 mapping,
1747 primary_channel.and_then(|i| channels.get(i)),
1748 &time_s,
1749 &rows.lines,
1750 )?;
1751 changes.extend(rows.marks.iter().map(|(r, w)| (*r, w.as_str())));
1752 changes.sort_by_key(|(row, _)| *row);
1755 let events = changes
1756 .into_iter()
1757 .map(|(row, word)| Event {
1758 time_s: time_s[row],
1759 name: word.to_owned(),
1760 })
1761 .collect();
1762 Ok(FlightLog {
1763 format: LogFormat::GenericTable,
1764 logger: "unknown".to_owned(),
1765 serial_number: None,
1766 firmware: None,
1767 flight_number: None,
1768 stated: Stated::default(),
1769 time_s: track_time_s,
1770 altitude_m,
1771 vertical_speed_m_s: None,
1772 temperature_k: None,
1773 battery_v: None,
1774 channels,
1775 events,
1776 notes,
1777 })
1778}
1779
1780fn primary_track(
1788 mapping: &Mapping,
1789 channel: Option<&Channel>,
1790 time_s: &[f64],
1791 lines: &[usize],
1792) -> Result<(Vec<f64>, Vec<f64>), LogError> {
1793 let Some(primary) = mapping.primary else {
1794 return Ok((Vec::new(), Vec::new()));
1795 };
1796 let column = mapping.columns.get(primary);
1797 let name = quoted(column.map_or("", |c| c.name.as_str()));
1798 let channel = match channel {
1799 None => {
1800 return Err(mapping_error(format!(
1801 "the primary track, {name}, holds no number"
1802 )));
1803 }
1804 Some(channel) if channel.quantity != Quantity::AltitudeAboveGround => {
1805 return Err(mapping_error(format!(
1806 "the primary track, {name}, holds values that contradict its unit, {}",
1807 quoted(column.and_then(|c| c.unit.as_deref()).unwrap_or_default())
1808 )));
1809 }
1810 Some(channel) => channel,
1811 };
1812 if let Some(step) = time_s.windows(2).position(|pair| pair[1] <= pair[0]) {
1813 let clock = mapping
1814 .columns
1815 .get(mapping.clock)
1816 .map_or("", |c| c.name.as_str());
1817 return Err(mapping_error(format!(
1818 "the primary track, {name}, needs a clock that goes forward, and the clock, {}, \
1819 doesn't on line {}: {} s, then {} s",
1820 quoted(clock),
1821 lines.get(step + 1).copied().unwrap_or_default(),
1822 time_s[step],
1823 time_s[step + 1]
1824 )));
1825 }
1826 Ok((channel.time_s.clone(), channel.values.clone()))
1827}
1828
1829#[cfg(test)]
1830mod tests {
1831 use super::*;
1832
1833 fn channel<'a>(log: &'a FlightLog, name: &str) -> &'a Channel {
1834 log.channels
1835 .iter()
1836 .find(|c| c.column == name)
1837 .unwrap_or_else(|| panic!("no {name}"))
1838 }
1839
1840 #[test]
1841 fn the_unit_table_and_its_names_agree() {
1842 assert_eq!(unit_spellings().count(), UNIT_TABLE.len());
1843 for (name, _) in UNIT_TABLE {
1844 assert_eq!(key(name), *name, "{name} isn't a key");
1845 assert_eq!(
1846 UNIT_TABLE.iter().filter(|(other, _)| other == name).count(),
1847 1,
1848 "{name} twice"
1849 );
1850 }
1851 }
1852
1853 #[test]
1855 fn units_convert_by_hand() {
1856 let cases = [
1857 ("ms", 1500.0, 1.5),
1858 ("us", 2_500_000.0, 2.5),
1859 ("ft", 100.0, 30.48),
1860 ("km", 1.5, 1500.0),
1861 ("ft/s", 10.0, 3.048),
1862 ("km/h", 36.0, 10.0),
1863 ("mph", 100.0, 44.704),
1864 ("kn", 3600.0, 1852.0),
1865 ("g", 2.0, 19.6133),
1866 ("mg", 1000.0, 9.80665),
1867 ("rad", std::f64::consts::PI, 180.0),
1868 ("rad/s", 1.0, RADIAN_DEG),
1869 ("degc", 25.0, 298.15),
1870 ("degf", 212.0, 373.15),
1871 ("f", 32.0, 273.15),
1872 ("mv", 3700.0, 3.7),
1873 ("ma", 250.0, 0.25),
1874 ("hpa", 1013.25, 101_325.0),
1875 ("kpa", 101.325, 101_325.0),
1876 ("bar", 1.01325, 101_325.0),
1877 ("atm", 1.0, 101_325.0),
1878 ("psi", 14.695_95, 101_325.0),
1879 ("inhg", 29.92, 101_320.76),
1880 ];
1881 for (unit, written, expected) in cases {
1882 let converted = unit_of(unit)
1883 .unwrap_or_else(|| panic!("{unit}"))
1884 .convert(written);
1885 assert!(
1886 (converted - expected).abs() < 1e-2 * expected.abs().max(1.0) * 1e-3,
1887 "{unit}: {written} → {converted}, expected {expected}"
1888 );
1889 }
1890 }
1891
1892 #[test]
1893 fn names_split_into_the_name_and_the_unit() {
1894 let cases: [(&str, &str, Option<&str>); 18] = [
1895 ("Time (s)", "Time", Some("s")),
1896 ("Alt AGL (ft)", "Alt AGL", Some("ft")),
1897 ("Altitude [m]", "Altitude", Some("m")),
1898 ("Temperature [deg C]", "Temperature", Some("degc")),
1899 ("Alt (m ASL)", "Alt ASL", Some("m")),
1900 ("Lon (° dec)", "Lon dec", Some("°")),
1901 ("Flight_Time_(s)", "Flight_Time", Some("s")),
1902 (
1903 "Baro_Altitude_AGL_(feet)",
1904 "Baro_Altitude_AGL",
1905 Some("feet"),
1906 ),
1907 ("time__us", "time", Some("us")),
1908 ("acc_y__m_s2", "acc_y", Some("m_s2")),
1909 ("pitch_rate_deg_per_s", "pitch_rate", Some("deg_per_s")),
1910 ("vel_north_meters_per_s", "vel_north", Some("meters_per_s")),
1911 ("Yacc_g", "Yacc", Some("g")),
1912 ("time(raw)", "time(raw)", None),
1913 ("flag_a", "flag_a", None),
1914 ("Load (grams)", "Load (grams)", None),
1915 ("Velocity_Up", "Velocity_Up", None),
1916 ("s", "s", None),
1917 ];
1918 for (written, name, unit) in cases {
1919 let (got_name, got_unit) = split_name(written);
1920 assert_eq!(
1921 (got_name.as_str(), got_unit.as_deref()),
1922 (name, unit),
1923 "{written}"
1924 );
1925 }
1926 }
1927
1928 #[test]
1929 fn a_column_is_read_only_where_name_and_unit_agree() {
1930 let cases: [(&str, Option<Quantity>); 28] = [
1931 ("Alt AGL (ft)", Some(Quantity::Altitude)),
1932 (
1933 "Baro_Altitude_AGL_(feet)",
1934 Some(Quantity::AltitudeAboveGround),
1935 ),
1936 ("alt_gps_m", Some(Quantity::GnssAltitude)),
1937 ("Lat (° dec)", Some(Quantity::Latitude)),
1938 ("Lon (° dec)", Some(Quantity::Longitude)),
1939 ("Tilt (deg)", Some(Quantity::Angle)),
1940 ("baro__kpa", Some(Quantity::Pressure)),
1941 ("Baro (Pa)", Some(Quantity::Pressure)),
1942 ("speed_baro(m/s)", Some(Quantity::Speed)),
1943 ("gyro_y__rad_s", Some(Quantity::AngularRate)),
1944 ("servo_current_milliamps", Some(Quantity::Current)),
1945 ("Temp (F)", Some(Quantity::Temperature)),
1946 ("vz_filtered_ms", None),
1947 ("z (m)", None),
1948 ("sensor_read_us", None),
1949 ("battery_voltage", None),
1950 ("Fin Position (deg)", Some(Quantity::Angle)),
1953 ("Apo_Volts", Some(Quantity::Voltage)),
1954 ("Mag X (G)", None),
1955 ("Load (g)", None),
1956 ("kalman_v", None),
1957 ("pyro_c", None),
1958 ("Roll rate (deg)", None),
1960 ("Alt error (m)", None),
1962 ("Altitude accuracy (m)", None),
1963 ("Diff press (Pa)", None),
1964 ("Baro pitot (Pa)", None),
1965 ("Alt AGL std (ft)", None),
1966 ];
1967 for (written, quantity) in cases {
1968 assert_eq!(infer(written).0.quantity, quantity, "{written}");
1969 }
1970 }
1971
1972 #[test]
1973 fn quoted_cells_hold_the_separator() {
1974 let got: Vec<String> = cells(r#" 1, "a,b" ,"say ""hi""","" "#, ',', usize::MAX)
1975 .into_iter()
1976 .map(Cow::into_owned)
1977 .collect();
1978 assert_eq!(got, ["1", "a,b", r#"say "hi""#, ""]);
1979 }
1980
1981 const TABLE: &str = "# an invented flight\n\
1982 Time (ms);Alt AGL (ft);Pressure (hPa);Phase;Note;Batt (V)\n\
1983 0;0;1000,5;PAD;;7,9\n\
1984 50;10;999,0;PAD;x;n/a\n\
1985 100;30;997,5;BOOST;;7,8\n\
1986 150;60;995,0;COAST;;7,8\n";
1987
1988 #[test]
1989 fn a_table_reads_by_its_header() {
1990 assert!(recognizes(TABLE));
1991 let log = read(TABLE).expect("reads");
1992 assert_eq!(log.format, LogFormat::GenericTable);
1993 assert!(log.time_s.is_empty() && log.altitude_m.is_empty());
1994 let altitude = channel(&log, "Alt AGL (ft)");
1995 assert_eq!(altitude.quantity, Quantity::Altitude);
1996 assert_eq!(altitude.time_s, [0.0, 0.05, 0.1, 0.15]);
1997 assert_eq!(altitude.values[3], 60.0 * FOOT_M);
1998 let pressure = channel(&log, "Pressure (hPa)");
1999 assert_eq!(pressure.values[0], 100_050.0);
2000 let names: Vec<&str> = log.events.iter().map(|e| e.name.as_str()).collect();
2001 assert_eq!(names, ["PAD", "BOOST", "COAST"]);
2002 assert_eq!(log.events[1].time_s, 0.1);
2003 assert!(
2004 log.channels
2005 .iter()
2006 .all(|c| c.column != "Note" && c.column != "Phase")
2007 );
2008 assert!(
2009 log.notes
2010 .iter()
2011 .any(|n| n.contains("\"Note\"") && n.contains("text"))
2012 );
2013 let battery = channel(&log, "Batt (V)");
2014 assert_eq!(battery.quantity, Quantity::Voltage);
2015 assert!(battery.values[1].is_nan());
2016 assert_eq!(battery.values[2], 7.8);
2017 assert!(
2018 log.notes
2019 .iter()
2020 .any(|n| n.contains("\"Batt (V)\"") && n.contains("1 cells"))
2021 );
2022 }
2023
2024 #[test]
2025 fn values_that_contradict_the_unit_keep_the_column_as_written() {
2026 let text = "t (s),Pressure (Pa),Temp (C)\n0,101000,20\n1,150000,21\n";
2027 let log = read(text).expect("reads");
2028 let pressure = channel(&log, "Pressure (Pa)");
2029 assert_eq!(pressure.quantity, Quantity::Unrecognized);
2030 assert_eq!(pressure.values, [101_000.0, 150_000.0]);
2031 assert!(log.notes.iter().any(|n| n.contains("150000 on line 3")));
2032 assert_eq!(channel(&log, "Temp (C)").quantity, Quantity::Temperature);
2033 let text = "t (s),Pressure (Pa)\n0,101000\n1,109999\n";
2035 assert_eq!(
2036 channel(&read(text).expect("reads"), "Pressure (Pa)").quantity,
2037 Quantity::Pressure
2038 );
2039 }
2040
2041 #[test]
2042 fn a_clock_whose_step_contradicts_its_unit_is_refused() {
2043 let text = "Time (s),Alt (m)\n0,0\n20,1\n40,3\n";
2046 assert!(read(text).is_ok());
2047 let text = "Time (s),Alt (m)\n0,0\n61,1\n122,3\n";
2048 match read(text) {
2049 Err(LogError::Unit { message, .. }) => assert!(message.contains("61"), "{message}"),
2050 other => panic!("expected a unit refusal, got {other:?}"),
2051 }
2052 let text = "Time (ms),Alt (m)\n0,0\n0.05,1\n0.1,3\n";
2053 assert!(matches!(read(text), Err(LogError::Unit { .. })));
2054 for text in [
2056 "Time (s),Alt (m)\n5,0\n",
2057 "Time (s),Alt (m)\n5,0\n5,1\n4,2\n",
2058 ] {
2059 match read(text) {
2060 Err(LogError::Unit { message, .. }) => {
2061 assert!(message.contains("never steps forward"), "{message}");
2062 }
2063 other => panic!("expected a unit refusal, got {other:?}"),
2064 }
2065 }
2066 }
2067
2068 #[test]
2069 fn rows_must_match_the_header() {
2070 let text = "Time (s),Alt (m)\n0,0\n1,1,1\n";
2071 match read(text) {
2072 Err(LogError::Syntax { line, message, .. }) => {
2073 assert_eq!(line, 3);
2074 assert!(message.contains("has more cells"), "{message}");
2075 }
2076 other => panic!("expected a syntax refusal, got {other:?}"),
2077 }
2078 let text = "Time (s),Alt (m)\n0,0\nend,1\n";
2079 assert!(matches!(read(text), Err(LogError::Syntax { line: 3, .. })));
2080 let text = "Time (s),Alt (m)\n0,0\n1,1e308\n";
2081 assert!(
2082 read(text).is_ok(),
2083 "a length that large passes the conversion"
2084 );
2085 let text = "Time (s),Alt (km)\n0,0\n1,1.7e308\n";
2086 assert!(matches!(read(text), Err(LogError::Syntax { line: 3, .. })));
2087 }
2088
2089 #[test]
2090 fn what_isnt_this_layout_is_not_recognized() {
2091 for text in [
2092 "alt,t\n0,0\n",
2093 "time,altitude\n0,0\n",
2094 "Time (s)\n0\n",
2095 "1,2,3\n4,5,6\n",
2096 "Time (s),Alt (m)\nnot a number,0\n",
2097 "Time (s),Alt (m)\n",
2098 "Burnout time (ms),1500,Apogee meters,300\n",
2099 ] {
2100 assert!(!recognizes(text), "{text:?}");
2101 assert!(matches!(read(text), Err(LogError::NotThisFormat { .. })));
2102 }
2103 }
2104
2105 #[test]
2106 fn a_mapping_reads_a_file_with_no_header() {
2107 let text = "0,100\n1,90\n2,80\n";
2108 let mapping = Mapping {
2109 header_line: None,
2110 separator: ',',
2111 decimal_comma: false,
2112 clock: 0,
2113 columns: vec![
2114 ColumnMapping {
2115 name: "time".to_owned(),
2116 quantity: None,
2117 unit: Some("s".to_owned()),
2118 skip: false,
2119 },
2120 ColumnMapping {
2121 name: "pressure".to_owned(),
2122 quantity: Some(Quantity::Pressure),
2123 unit: Some("kPa".to_owned()),
2124 skip: false,
2125 },
2126 ],
2127 primary: None,
2128 clock_wrap: None,
2129 rows: RowForm::Cells,
2130 };
2131 let log = read_with(text, &mapping).expect("reads");
2132 assert_eq!(
2133 channel(&log, "pressure").values,
2134 [100_000.0, 90_000.0, 80_000.0]
2135 );
2136 let mut wrong = mapping.clone();
2137 wrong.columns[1].unit = Some("ft".to_owned());
2138 match read_with(text, &wrong) {
2139 Err(LogError::Mapping { message }) => {
2140 assert!(message.contains("measures a length"), "{message}");
2141 }
2142 other => panic!("expected a mapping refusal, got {other:?}"),
2143 }
2144 let refusal = |wrong: &Mapping| match read_with(text, wrong) {
2145 Err(LogError::Mapping { message }) => message,
2146 other => panic!("expected a mapping refusal, got {other:?}"),
2147 };
2148 let mut wrong = mapping.clone();
2149 wrong.columns[0].unit = Some("m".to_owned());
2150 assert!(refusal(&wrong).contains("no unit of time"));
2151 let mut wrong = mapping.clone();
2152 wrong.clock = 2;
2153 assert!(refusal(&wrong).contains("the clock is column 2, of 2"));
2154 let mut wrong = mapping.clone();
2155 wrong.columns[1].unit = Some(format!("furlongs{}", " ".repeat(10_000)));
2156 let message = refusal(&wrong);
2157 assert!(message.contains("isn't one the reader knows") && message.len() < 200);
2158 let mut wrong = mapping.clone();
2159 wrong.separator = '.';
2160 assert!(refusal(&wrong).contains("isn't one of"));
2161 let mut wrong = mapping.clone();
2162 wrong.decimal_comma = true;
2163 assert!(refusal(&wrong).contains("a decimal comma needs"));
2164 let mut semicolons = mapping;
2166 semicolons.separator = ';';
2167 let log = read_with("sep=;\n0;100\n1;90\n", &semicolons).expect("reads");
2168 assert_eq!(channel(&log, "pressure").values, [100_000.0, 90_000.0]);
2169 }
2170
2171 #[test]
2174 fn a_thousands_separator_is_never_a_decimal_mark() {
2175 let text = "Time (s)\tAlt (ft)\n0.00\t0\n0.05\t1,250\n0.10\t980\n";
2177 let altitude = channel(&read(text).expect("reads"), "Alt (ft)")
2178 .values
2179 .clone();
2180 assert!(altitude[1].is_nan());
2181 assert_eq!(altitude[2], 980.0 * FOOT_M);
2182 let text = "Time (ms);Alt (ft)\n0;0\n50;1,250\n100;2,500\n";
2184 let log = read(text).expect("reads");
2185 assert!(
2186 channel(&log, "Alt (ft)").values[1..]
2187 .iter()
2188 .all(|v| v.is_nan())
2189 );
2190 assert!(log.notes.iter().any(|n| n.contains("\"1,250\": gaps")));
2191 let text = "Time (s);Alt (ft)\n0;0\n0,05;1250\n0,10;2,5\n";
2193 let mut by_header = mapping(text).expect("a header");
2194 assert!(by_header.decimal_comma);
2195 let altitude = channel(&read(text).expect("reads"), "Alt (ft)")
2196 .values
2197 .clone();
2198 assert_eq!(altitude[1..], [1250.0 * FOOT_M, 2.5 * FOOT_M]);
2199 let text = "Time (s);Alt (ft)\n0;0\n0,05;1.250\n0,10;2,5\n";
2201 assert!(matches!(read(text), Err(LogError::Syntax { line: 3, .. })));
2202 by_header.header_line = Some(1);
2203 let altitude = channel(&read_with(text, &by_header).expect("reads"), "Alt (ft)")
2204 .values
2205 .clone();
2206 assert!(altitude[1].is_nan());
2207 assert_eq!(altitude[2], 2.5 * FOOT_M);
2208 let text = "Time (s),Alt (ft)\n0,0\n1,\"1,250\"\n";
2210 assert!(channel(&read(text).expect("reads"), "Alt (ft)").values[1].is_nan());
2211 }
2212
2213 #[test]
2216 fn every_columns_note_is_kept() {
2217 let names: Vec<String> = (1..=30).map(|i| format!("c{i}")).collect();
2218 let text = format!(
2219 "Time (s),{},Pressure (Pa)\n0,{}101000\n1,{}150000\n",
2220 names.join(","),
2221 "1,".repeat(30),
2222 "2,".repeat(30)
2223 );
2224 let log = read(&text).expect("reads");
2225 for name in &names {
2226 assert!(
2227 log.notes
2228 .iter()
2229 .any(|n| n.contains(&format!("{name:?} states no unit"))),
2230 "{name}"
2231 );
2232 }
2233 assert!(
2234 log.notes
2235 .iter()
2236 .any(|n| n.contains("\"Pressure (Pa)\"'s values contradict"))
2237 );
2238 assert!(!log.notes.iter().any(|n| n.contains("more notes")));
2239 }
2240
2241 #[test]
2243 fn events_from_two_columns_are_in_the_files_order() {
2244 let text = "Time (s),State,Mode\n0,PAD,A\n1,BOOST,A\n2,BOOST,B\n3,BOOST,B\n";
2245 let log = read(text).expect("reads");
2246 let events: Vec<(f64, &str)> = log
2247 .events
2248 .iter()
2249 .map(|e| (e.time_s, e.name.as_str()))
2250 .collect();
2251 assert_eq!(
2252 events,
2253 [(0.0, "PAD"), (0.0, "A"), (1.0, "BOOST"), (2.0, "B")]
2254 );
2255 let rows: String = (0..200).map(|i| format!("{i},{}\n", i % 2)).collect();
2257 let log = read(&format!(
2258 "Time (s),Mode\n{}",
2259 rows.replace(",0\n", ",A\n").replace(",1\n", ",B\n")
2260 ))
2261 .expect("reads");
2262 assert_eq!(log.events.len(), MAX_EVENTS_PER_COLUMN);
2263 assert!(log.notes.iter().any(|n| n.contains("136 more times")));
2264 let text = "Time (s),Model\n0,EasyMini\n1,EasyMini\n";
2266 let log = read(text).expect("reads");
2267 assert!(log.events.is_empty());
2268 assert!(log.notes.iter().any(|n| n.contains("\"Model\" holds text")));
2269 }
2270
2271 proptest::proptest! {
2272 #[test]
2276 fn any_text_is_read_or_refused(
2277 text in "(sep=[;,]\n)?(Time \\(m?s\\)[,;\t](Alt|Temp|Press|v)[_ ]?\\(?(ft|m|C|kPa|m/s|g)?\\)?[,;\t]?[a-z\"]{0,4}\n)?\
2278 ([-0-9.,;\t eE\"a-z]{0,24}\n){0,12}"
2279 ) {
2280 if let Ok(log) = read(&text) {
2281 for channel in &log.channels {
2282 proptest::prop_assert_eq!(channel.time_s.len(), channel.values.len());
2283 if let Some((low, high)) = dimension_of(channel.quantity)
2284 .and_then(|d| bounds_of(channel.quantity, d))
2285 {
2286 for value in &channel.values {
2287 proptest::prop_assert!(
2288 value.is_nan() || (low..=high).contains(value),
2289 "{value} in {:?}", channel.quantity
2290 );
2291 }
2292 }
2293 }
2294 }
2295 }
2296 }
2297
2298 #[test]
2299 fn the_mapping_round_trips_through_json() {
2300 let mapping = mapping(TABLE).expect("a header");
2301 let json = serde_json::to_string(&mapping).expect("writes");
2302 let back: Mapping = serde_json::from_str(&json).expect("reads");
2303 assert_eq!(back, mapping);
2304 let by_mapping = read_with(TABLE, &back).expect("reads");
2305 let by_header = read(TABLE).expect("reads");
2306 let json = |log: &FlightLog| serde_json::to_value(&log.channels).expect("writes");
2308 assert_eq!(json(&by_mapping), json(&by_header));
2309 assert_eq!(by_mapping.events, by_header.events);
2310 }
2311
2312 fn column(name: &str, quantity: Option<Quantity>, unit: Option<&str>) -> ColumnMapping {
2313 ColumnMapping {
2314 name: name.to_owned(),
2315 quantity,
2316 unit: unit.map(str::to_owned),
2317 skip: false,
2318 }
2319 }
2320
2321 fn headerless(columns: Vec<ColumnMapping>) -> Mapping {
2322 Mapping {
2323 header_line: None,
2324 separator: ',',
2325 decimal_comma: false,
2326 clock: 0,
2327 columns,
2328 primary: None,
2329 clock_wrap: None,
2330 rows: RowForm::Cells,
2331 }
2332 }
2333
2334 fn mapping_refusal(text: &str, mapping: &Mapping) -> String {
2335 match read_with(text, mapping) {
2336 Err(LogError::Mapping { message }) => message,
2337 other => panic!("expected a mapping refusal, got {other:?}"),
2338 }
2339 }
2340
2341 #[test]
2344 fn a_clock_that_turns_over_runs_on() {
2345 let turn = 4_294_967_296.0_f64;
2346 let counts = [
2347 turn - 40_000.0,
2348 turn - 20_000.0,
2349 0.0,
2350 20_000.0,
2351 turn / 2.0 + 10_000.0,
2352 ];
2353 let written = [
2354 counts[0] + 9.0 * turn,
2355 counts[1] + 9.0 * turn,
2356 counts[2],
2357 counts[3],
2358 counts[4] + 9.0 * turn,
2359 ];
2360 let text: String = written
2361 .iter()
2362 .enumerate()
2363 .map(|(k, w)| format!("{w},{}\n", 90_000 + k))
2364 .collect();
2365 let mut mapping = headerless(vec![
2366 column("clock", None, Some("us")),
2367 column("pressure", Some(Quantity::Pressure), Some("Pa")),
2368 ]);
2369 mapping.clock_wrap = Some(turn);
2370 let log = read_with(&text, &mapping).expect("reads");
2371 let times = &channel(&log, "pressure").time_s;
2372 let expected = [
2373 (turn - 40_000.0) / 1e6,
2374 (turn - 20_000.0) / 1e6,
2375 turn / 1e6,
2376 (turn + 20_000.0) / 1e6,
2377 (turn + turn / 2.0 + 10_000.0) / 1e6,
2378 ];
2379 assert_eq!(times.as_slice(), expected);
2380 assert!(log.notes.iter().any(|n| n.contains(
2381 "counts modulo 4294967296: 3 values written past it taken modulo it, and it turns over \
2382 once"
2383 )));
2384 mapping.clock_wrap = None;
2386 let log = read_with(&text, &mapping).expect("reads");
2387 assert!(log.notes.iter().any(|n| n.contains("steps back 1 times")));
2388 for bad in [0.0, -1.0, f64::NAN, f64::INFINITY] {
2389 mapping.clock_wrap = Some(bad);
2390 assert!(mapping_refusal(&text, &mapping).contains("isn't a count above 0"));
2391 }
2392 }
2393
2394 #[test]
2395 fn rows_written_as_lists_drop_their_brackets() {
2396 let mut mapping = headerless(vec![
2397 column("clock", None, Some("ms")),
2398 column("pressure", Some(Quantity::Pressure), Some("Pa")),
2399 column("fix", None, None),
2400 ]);
2401 mapping.rows = RowForm::List;
2402 let text = "[0, 101325, True]\n[500, 101300, True]\n";
2403 let log = read_with(text, &mapping).expect("reads");
2404 assert_eq!(channel(&log, "pressure").values, [101_325.0, 101_300.0]);
2405 assert_eq!(channel(&log, "pressure").time_s, [0.0, 0.5]);
2406 assert!(
2407 log.notes
2408 .iter()
2409 .any(|n| n.contains("\"fix\" holds text, not numbers, such as \"True\""))
2410 );
2411 for row in ["1000, 101200, True]", "[1000, 101200, True"] {
2412 match read_with(&format!("{text}{row}\n"), &mapping) {
2413 Err(LogError::Syntax { line, message, .. }) => {
2414 assert_eq!(line, 3);
2415 assert!(message.contains("isn't a list"), "{message}");
2416 }
2417 other => panic!("expected a refusal, got {other:?}"),
2418 }
2419 }
2420 }
2421
2422 #[test]
2423 fn stamped_rows_take_their_stamp_as_the_clock() {
2424 let mut mapping = headerless(vec![
2425 column("stamp", None, Some("s")),
2426 column("x", Some(Quantity::Acceleration), Some("m/s^2")),
2427 column("y", Some(Quantity::Acceleration), Some("m/s^2")),
2428 column("z", Some(Quantity::Acceleration), Some("m/s^2")),
2429 ]);
2430 mapping.rows = RowForm::Stamped;
2431 let text = "2022-10-13 23:59:59.875: (0.5, -0.25, 9.75)\n\
2432 ===== MOTION DETECTED =====\n\
2433 2022-10-14 00:00:00.125000: (1, 2, 30)\n\
2434 2022-10-14 00:00:01: (0, 0, 9.8)\n\
2435 -- end --\n";
2436 let log = read_with(text, &mapping).expect("reads");
2437 let midnight = 19_279.0 * 86_400.0;
2439 assert_eq!(
2440 channel(&log, "z").time_s,
2441 [midnight - 0.125, midnight + 0.125, midnight + 1.0]
2442 );
2443 assert_eq!(channel(&log, "x").values, [0.5, 1.0, 0.0]);
2444 assert_eq!(channel(&log, "z").values, [9.75, 30.0, 9.8]);
2445 assert_eq!(log.events.len(), 1);
2446 assert_eq!(log.events[0].name, "MOTION DETECTED");
2447 assert_eq!(log.events[0].time_s, midnight + 0.125);
2448 assert!(
2449 log.notes
2450 .iter()
2451 .any(|n| n.contains("1 line with no stamp: each a mark"))
2452 );
2453 assert!(
2454 log.notes
2455 .iter()
2456 .any(|n| n.contains("1 line with no stamp after the last row"))
2457 );
2458 for bad in [
2460 "2022-02-29 00:00:02: (0, 0, 9.8)",
2461 "2022-10-14 24:00:02: (0, 0, 9.8)",
2462 "2022-10-14 00:00:02. (0, 0, 9.8)",
2463 "2022-10-14 00:00:02 (0, 0, 9.8)",
2464 "2022-10-14 00:00:02.1234567890: (0, 0, 9.8)",
2465 ] {
2466 match read_with(&format!("{text}{bad}\n"), &mapping) {
2467 Err(LogError::Syntax { line, message, .. }) => {
2468 assert_eq!(line, 6, "{bad}");
2469 assert!(message.contains("stamp isn't a date"), "{message}");
2470 }
2471 other => panic!("{bad}: expected a refusal, got {other:?}"),
2472 }
2473 }
2474 let mut one = headerless(vec![
2476 column("stamp", None, Some("s")),
2477 column("pressure", Some(Quantity::Pressure), Some("hPa")),
2478 ]);
2479 one.rows = RowForm::Stamped;
2480 let log = read_with(
2481 "2024-02-29 12:00:00: 1005.5\n2024-02-29 12:00:00.5: 1005.25\n",
2482 &one,
2483 )
2484 .expect("reads");
2485 assert_eq!(channel(&log, "pressure").values, [100_550.0, 100_525.0]);
2486 let many = format!(
2488 "{}2024-02-29 12:00:00: 1005.5\n{}2024-02-29 12:00:01: 1005.25\nend\nend\n",
2489 "mark\n".repeat(MAX_MARKS - 2),
2490 "mark\n".repeat(5)
2491 );
2492 let log = read_with(&many, &one).expect("reads");
2493 assert_eq!(log.events.len(), MAX_MARKS);
2494 assert!(log.notes.iter().any(|n| n.contains(&format!(
2495 "{} lines with no stamp: each a mark, an event at the next row's time, the first \
2496 {MAX_MARKS} kept",
2497 MAX_MARKS + 3
2498 ))));
2499 assert!(
2500 log.notes
2501 .iter()
2502 .any(|n| n.contains("2 lines with no stamp after the last row"))
2503 );
2504 let mut wrong = one.clone();
2505 wrong.columns[0].unit = Some("ms".to_owned());
2506 assert!(mapping_refusal("", &wrong).contains("column 0, in `s`"));
2507 let mut wrong = one;
2508 wrong.clock_wrap = Some(60.0);
2509 assert!(mapping_refusal("", &wrong).contains("with no turn"));
2510 }
2511
2512 #[test]
2513 fn columns_left_out_make_no_channel() {
2514 let mut mapping = headerless(vec![
2515 column("time", None, Some("s")),
2516 column("altitude", Some(Quantity::AltitudeAboveGround), Some("ft")),
2517 column("beside 1", None, None),
2518 column("beside 2", Some(Quantity::Pressure), Some("ft")),
2519 ]);
2520 mapping.columns[2].skip = true;
2521 mapping.columns[3].skip = true;
2522 let text = "0,0,Name,12\n0.5,100,,\n1,180,,\n";
2523 let log = read_with(text, &mapping).expect("reads");
2524 assert_eq!(log.channels.len(), 1);
2525 assert_eq!(log.channels[0].column, "altitude");
2526 assert!(
2527 log.notes
2528 .iter()
2529 .any(|n| n == "the mapping leaves out 2 of the 4 columns")
2530 );
2531 assert!(!log.notes.iter().any(|n| n.contains("Name")));
2532 assert!(matches!(
2534 read_with("0,0,Name\n", &mapping),
2535 Err(LogError::Syntax { line: 1, .. })
2536 ));
2537 mapping.columns[3].skip = false;
2539 assert!(mapping_refusal(text, &mapping).contains("\"beside 2\""));
2540 mapping.columns[3].skip = true;
2541 mapping.columns[0].skip = true;
2542 assert!(mapping_refusal(text, &mapping).contains("the mapping leaves out"));
2543 }
2544
2545 #[test]
2548 fn a_mappings_options_default_and_round_trip() {
2549 let old = r#"{"header_line":null,"separator":",","decimal_comma":false,"clock":0,"columns":[{"name":"t","quantity":null,"unit":"s"}]}"#;
2550 let mapping: Mapping = serde_json::from_str(old).expect("parses");
2551 assert_eq!(mapping.clock_wrap, None);
2552 assert_eq!(mapping.primary, None);
2553 assert_eq!(mapping.rows, RowForm::Cells);
2554 assert!(!mapping.columns[0].skip);
2555 assert_eq!(serde_json::to_string(&mapping).expect("writes"), old);
2556 let new = r#"{"header_line":null,"separator":",","decimal_comma":false,"clock":0,"primary":1,"columns":[{"name":"t","quantity":null,"unit":"s"},{"name":"x","quantity":null,"unit":null,"skip":true}],"clock_wrap":4294967296.0,"rows":"stamped"}"#;
2557 let mapping: Mapping = serde_json::from_str(new).expect("parses");
2558 assert_eq!(mapping.clock_wrap, Some(4_294_967_296.0));
2559 assert_eq!(mapping.primary, Some(1));
2560 assert_eq!(mapping.rows, RowForm::Stamped);
2561 assert!(mapping.columns[1].skip);
2562 assert_eq!(serde_json::to_string(&mapping).expect("writes"), new);
2563 let list: Mapping = serde_json::from_str(&new.replace("stamped", "list")).expect("parses");
2564 assert_eq!(list.rows, RowForm::List);
2565 }
2566}