Skip to main content

hpr_flightdata/
archive.rs

1//! Zip archives of a flight's files, as flyers share them: a Blue Raven's high-rate and low-rate
2//! files with the app's summary, or a folder of everything a flight left behind.
3//!
4//! An archive isn't a log. [`unpack`] takes its files out, in memory, and each is then read as a
5//! file on its own ([`import::read`](crate::import::read)); an archive inside an archive is one of
6//! its files like any other, and is not opened. Nothing is written anywhere, so a file's name is
7//! kept as the archive gives it, folders and all, and is only a label.
8//!
9//! An archive comes from someone else, so what comes out of it is bounded: at most
10//! [`MAX_UNPACKED_BYTES`] of decompressed bytes, counted as they come out rather than as the
11//! archive states them. (The zip reader's own record of the archive's directory, a few times the
12//! directory's size, is on top.) Two files of one name, which the zip reader would silently keep
13//! one of, are refused. A file is read with only the methods zip archives use for flight logs,
14//! stored and deflated; an encrypted file, another method or a damaged file is refused with its
15//! reason. The zip format is PKWARE's *APPNOTE.TXT*, version 6.3.10.
16//!
17//! ```
18//! use std::io::{Cursor, Write};
19//!
20//! use hpr_flightdata::archive;
21//!
22//! // An invented archive of one PerfectFlite export.
23//! let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
24//! zip.start_file("flight/export.csv", zip::write::SimpleFileOptions::default())?;
25//! zip.write_all(b"Time,Altitude,Velocity\n0.00,0,0\n0.05,1,8\n0.10,3,30\n")?;
26//! let bytes = zip.finish()?.into_inner();
27//!
28//! let members = archive::unpack(&bytes)?;
29//! assert_eq!(members[0].name, "flight/export.csv");
30//! let log = hpr_flightdata::import::read(&members[0].bytes)?;
31//! assert_eq!(log.time_s.len(), 3);
32//! # Ok::<(), Box<dyn std::error::Error>>(())
33//! ```
34
35use std::io::{Cursor, Read};
36
37use serde::{Deserialize, Serialize};
38
39use crate::error::LogError;
40
41/// The most bytes [`unpack`] takes out of one archive, 256 MiB (268 MB): about eight times the
42/// largest of the project's sample archives, which unpacks to 32 MB, and the limit the `.hprz`
43/// design container holds to.
44pub const MAX_UNPACKED_BYTES: u64 = 256 * 1024 * 1024;
45
46/// The format's name in errors.
47const FORMAT: &str = "zip archive";
48
49/// One file taken out of an archive.
50#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
51#[non_exhaustive]
52pub struct Member {
53    /// The file's name as the archive gives it, folders before it separated by `/`.
54    pub name: String,
55    /// The file's bytes, decompressed.
56    pub bytes: Vec<u8>,
57}
58
59/// Whether the bytes open as a zip archive does: a local file header, or the end of an empty
60/// archive's central directory (APPNOTE.TXT 4.3.7, 4.3.16).
61pub fn is_archive(bytes: &[u8]) -> bool {
62    matches!(bytes, [b'P', b'K', 3, 4, ..] | [b'P', b'K', 5, 6, ..])
63}
64
65/// The files in a zip archive, in its central directory's order, folders' own entries left out;
66/// [`unpack_within`] with [`MAX_UNPACKED_BYTES`].
67///
68/// # Errors
69///
70/// As [`unpack_within`].
71pub fn unpack(bytes: &[u8]) -> Result<Vec<Member>, LogError> {
72    unpack_within(bytes, MAX_UNPACKED_BYTES)
73}
74
75/// The files in a zip archive, decompressing at most `max_unpacked_bytes` out of it.
76///
77/// # Errors
78///
79/// [`LogError::NotThisFormat`] when the bytes don't open as a zip archive; [`LogError::Archive`]
80/// when the zip reader can't read its central directory, the directory holds more entries than
81/// the reader keeps (two of one name), an entry's name can't be read, an entry is a symbolic link
82/// or a folder's entry that holds bytes, a file is encrypted, compressed by a method other than stored or deflated, or damaged
83/// (its CRC-32 fails, or it holds more bytes than it states), or the archive decompresses to more
84/// than `max_unpacked_bytes`.
85pub fn unpack_within(bytes: &[u8], max_unpacked_bytes: u64) -> Result<Vec<Member>, LogError> {
86    if !is_archive(bytes) {
87        return Err(LogError::NotThisFormat {
88            format: FORMAT,
89            message: "its bytes don't open as a zip archive's, `PK` then 3, 4".to_owned(),
90        });
91    }
92    let refused = |message: String| LogError::Archive { message };
93    let mut archive = zip::ZipArchive::new(Cursor::new(bytes))
94        .map_err(|error| refused(format!("its central directory can't be read: {error}")))?;
95    // The zip reader keeps one entry of each name and drops the others without a word, so the
96    // central directory's records are counted here: more records than entries means two share a
97    // name, and one of them would be lost.
98    let start = usize::try_from(archive.central_directory_start()).unwrap_or(usize::MAX);
99    let records = central_records(bytes, start);
100    if records != archive.len() {
101        return Err(refused(format!(
102            "its central directory holds {records} entries, and the zip reader keeps {}. Two \
103             entries have the same name, and one would be lost, or the directory is damaged",
104            archive.len()
105        )));
106    }
107    let mut members = Vec::new();
108    let mut used = 0u64;
109    for index in 0..archive.len() {
110        let mut entry = archive
111            .by_index(index)
112            .map_err(|error| refused(format!("entry {index} can't be opened: {error}")))?;
113        let name = entry
114            .name()
115            .map_err(|error| refused(format!("entry {index}'s name can't be read: {error}")))?
116            .into_owned();
117        if entry.is_symlink() {
118            return Err(refused(format!(
119                "entry {index}, {:?}, is a symbolic link, not a file",
120                shortened(&name)
121            )));
122        }
123        let left = max_unpacked_bytes.saturating_sub(used);
124        let mut content = Vec::new();
125        // One byte past what is left, so that an entry filling the budget exactly is still known
126        // to have overrun it. The archive's stated sizes aren't trusted: the bytes are counted as
127        // they come out.
128        entry
129            .by_ref()
130            .take(left.saturating_add(1))
131            .read_to_end(&mut content)
132            .map_err(|error| {
133                refused(format!(
134                    "entry {index}, {:?}, can't be decompressed: {error}",
135                    shortened(&name)
136                ))
137            })?;
138        if content.len() as u64 > left {
139            return Err(refused(format!(
140                "it decompresses to more than {max_unpacked_bytes} bytes, the most taken out of \
141                 one archive"
142            )));
143        }
144        used += content.len() as u64;
145        if entry.is_dir() {
146            // A folder's entry holds nothing: one that holds bytes, whatever size it states,
147            // would be lost.
148            if entry.size() > 0 || !content.is_empty() {
149                return Err(refused(format!(
150                    "entry {index}, {:?}, is a folder's entry that holds bytes",
151                    shortened(&name)
152                )));
153            }
154            continue;
155        }
156        // `read_to_end` grows by doubling: the slack goes back before the bytes are kept.
157        content.shrink_to_fit();
158        members.push(Member {
159            name,
160            bytes: content,
161        });
162    }
163    Ok(members)
164}
165
166/// How many records a zip archive's central directory holds from `start`: each begins with the
167/// signature `PK\x01\x02` and is 46 bytes, then its name, extra field and comment, whose lengths
168/// are the little-endian `u16`s at 28, 30 and 32 (APPNOTE.TXT 6.3.10, section 4.3.12).
169fn central_records(bytes: &[u8], start: usize) -> usize {
170    const SIGNATURE: &[u8] = b"PK\x01\x02";
171    const FIXED: usize = 46;
172    let u16_at = |at: usize| usize::from(u16::from_le_bytes([bytes[at], bytes[at + 1]]));
173    let mut count = 0;
174    let mut at = start;
175    while let Some(record) = bytes.get(at..at.saturating_add(FIXED))
176        && record.starts_with(SIGNATURE)
177    {
178        count += 1;
179        // In range: the fixed part was just read.
180        at += FIXED + u16_at(at + 28) + u16_at(at + 30) + u16_at(at + 32);
181    }
182    count
183}
184
185/// A name cut to 60 characters for an error, so a hostile archive's 64 KiB name isn't copied
186/// into every message.
187fn shortened(name: &str) -> String {
188    const MOST: usize = 60;
189    if name.chars().count() <= MOST {
190        name.to_owned()
191    } else {
192        name.chars().take(MOST).collect::<String>() + "…"
193    }
194}
195
196#[cfg(test)]
197mod tests {
198    use std::io::Write;
199
200    use zip::write::SimpleFileOptions;
201
202    use super::*;
203
204    /// An archive of the given files, each deflated.
205    fn archive(files: &[(&str, &[u8])]) -> Vec<u8> {
206        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
207        for (name, bytes) in files {
208            zip.start_file(*name, SimpleFileOptions::default())
209                .expect("an in-memory archive takes a file");
210            zip.write_all(bytes).expect("an in-memory write succeeds");
211        }
212        zip.finish()
213            .expect("an in-memory archive finishes")
214            .into_inner()
215    }
216
217    fn message(result: Result<Vec<Member>, LogError>) -> String {
218        match result {
219            Err(LogError::Archive { message }) => message,
220            other => panic!("expected an archive refusal, got {other:?}"),
221        }
222    }
223
224    #[test]
225    fn files_come_out_in_order_with_their_names_and_folders_left_out() {
226        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
227        zip.add_directory("Data/", SimpleFileOptions::default())
228            .expect("a folder entry is written");
229        for (name, bytes) in [("Data/b.csv", &b"2"[..]), ("Data/a.csv", b"1")] {
230            zip.start_file(name, SimpleFileOptions::default())
231                .expect("a file entry is written");
232            zip.write_all(bytes).expect("an in-memory write succeeds");
233        }
234        let bytes = zip.finish().expect("finished").into_inner();
235        let members = unpack(&bytes).expect("a plain archive unpacks");
236        let names: Vec<&str> = members.iter().map(|member| member.name.as_str()).collect();
237        assert_eq!(names, ["Data/b.csv", "Data/a.csv"]);
238        assert_eq!(members[1].bytes, b"1");
239    }
240
241    #[test]
242    fn an_archive_inside_is_a_file_like_any_other() {
243        let inner = archive(&[("log.csv", b"x")]);
244        let members = unpack(&archive(&[("inner.zip", &inner)])).expect("unpacks");
245        assert_eq!(members.len(), 1);
246        assert_eq!(members[0].bytes, inner);
247    }
248
249    #[test]
250    fn the_budget_counts_the_bytes_that_come_out() {
251        let bytes = archive(&[("a", &[0u8; 600][..]), ("b", &[0u8; 400][..])]);
252        assert!(unpack_within(&bytes, 1000).is_ok());
253        // One byte short: the second file overruns it, though zeros compress to a few bytes.
254        let refusal = message(unpack_within(&bytes, 999));
255        assert!(refusal.contains("more than 999 bytes"), "{refusal}");
256    }
257
258    #[test]
259    fn a_stated_size_smaller_than_the_data_does_not_fool_the_budget() {
260        // A file of 4,000 zeros whose central directory states 10 bytes, uncompressed: a bomb's
261        // trick. Its size fields are at offset 24 of the central record and 22 of the local one.
262        // The zip reader gives no byte past the stated size: the read that crosses it fails, long
263        // before the budget; were it to give them, the budget counts what comes out.
264        let mut bytes = archive(&[("bomb", &[0u8; 4000][..])]);
265        let central = bytes
266            .windows(4)
267            .position(|window| window == b"PK\x01\x02")
268            .expect("the archive has a central directory");
269        bytes[central + 24..central + 28].copy_from_slice(&10u32.to_le_bytes());
270        bytes[22..26].copy_from_slice(&10u32.to_le_bytes());
271        let refusal = message(unpack_within(&bytes, 1000));
272        assert!(
273            refusal.contains("\"bomb\", can't be decompressed")
274                && refusal.contains("larger than its declared uncompressed size"),
275            "{refusal}"
276        );
277    }
278
279    #[test]
280    fn two_files_of_one_name_are_refused() {
281        // Written under two names of one length, then the second renamed to the first, in its
282        // local header and its central record alike.
283        let mut bytes = archive(&[("a.csv", b"1"), ("b.csv", b"2")]);
284        let positions: Vec<usize> = bytes
285            .windows(5)
286            .enumerate()
287            .filter(|(_, window)| *window == b"b.csv")
288            .map(|(at, _)| at)
289            .collect();
290        assert_eq!(positions.len(), 2);
291        for at in positions {
292            bytes[at] = b'a';
293        }
294        let refusal = message(unpack(&bytes));
295        assert!(refusal.contains("same name"), "{refusal}");
296    }
297
298    /// An archive of one file stored, not compressed.
299    fn stored(name: &str, bytes: &[u8]) -> Vec<u8> {
300        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
301        let options =
302            SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
303        zip.start_file(name, options)
304            .expect("a stored file is written");
305        zip.write_all(bytes).expect("an in-memory write succeeds");
306        zip.finish().expect("finished").into_inner()
307    }
308
309    /// The offset of the central directory's first record.
310    fn central(bytes: &[u8]) -> usize {
311        bytes
312            .windows(4)
313            .position(|window| window == b"PK\x01\x02")
314            .expect("the archive has a central directory")
315    }
316
317    #[test]
318    fn a_damaged_file_is_refused_by_its_checksum() {
319        // A stored file's bytes follow the 30-byte local header and the 5-byte name as they are,
320        // so a flipped byte there reaches the CRC-32 and nothing else.
321        let mut bytes = stored("a.csv", b"Time,Altitude\n0,0\n");
322        bytes[30 + 5] ^= 0x01;
323        let refusal = message(unpack(&bytes));
324        assert!(refusal.contains("can't be decompressed"), "{refusal}");
325        assert!(refusal.to_lowercase().contains("checksum"), "{refusal}");
326    }
327
328    #[test]
329    fn a_symbolic_link_is_refused() {
330        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
331        zip.add_symlink("link.csv", "/etc/passwd", SimpleFileOptions::default())
332            .expect("a link is written");
333        let bytes = zip.finish().expect("finished").into_inner();
334        let refusal = message(unpack(&bytes));
335        assert!(refusal.contains("symbolic link"), "{refusal}");
336    }
337
338    #[test]
339    fn an_encrypted_file_is_refused() {
340        // Bit 0 of the general-purpose flags, at offset 6 of the local header and 8 of the
341        // central record, marks a file encrypted (APPNOTE.TXT 4.4.4).
342        let mut bytes = stored("a.csv", b"0,0\n");
343        let at = central(&bytes);
344        bytes[6] |= 1;
345        bytes[at + 8] |= 1;
346        let refusal = message(unpack(&bytes));
347        assert!(refusal.contains("assword"), "{refusal}");
348    }
349
350    #[test]
351    fn a_method_other_than_stored_or_deflated_is_refused() {
352        // The method, at offset 8 of the local header and 10 of the central record, set to 12,
353        // bzip2 (APPNOTE.TXT 4.4.5), which the zip reader is built without.
354        let mut bytes = stored("a.csv", b"0,0\n");
355        let at = central(&bytes);
356        bytes[8..10].copy_from_slice(&12u16.to_le_bytes());
357        bytes[at + 10..at + 12].copy_from_slice(&12u16.to_le_bytes());
358        let refusal = message(unpack(&bytes));
359        assert!(refusal.contains("not supported"), "{refusal}");
360    }
361
362    #[test]
363    fn a_folder_entry_that_holds_bytes_is_refused_whatever_size_it_states() {
364        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
365        zip.start_file("dd/", SimpleFileOptions::default())
366            .expect("an entry named as a folder is written");
367        zip.write_all(&[b'x'; 500])
368            .expect("an in-memory write succeeds");
369        let mut bytes = zip.finish().expect("finished").into_inner();
370        // As written, it states its 500 bytes.
371        assert!(message(unpack(&bytes)).contains("folder's entry that holds bytes"));
372        // With its stated size, at offset 24 of the central record, set to 0, it still holds them,
373        // and the zip reader refuses the first byte past the 0 it states.
374        let at = central(&bytes);
375        bytes[at + 24..at + 28].copy_from_slice(&0u32.to_le_bytes());
376        let refusal = message(unpack(&bytes));
377        assert!(
378            refusal.contains("\"dd/\", can't be decompressed")
379                && refusal.contains("larger than its declared uncompressed size"),
380            "{refusal}"
381        );
382    }
383
384    #[test]
385    fn bytes_that_are_not_an_archive_are_not_this_format() {
386        assert!(matches!(
387            unpack(b"Time,Altitude\n"),
388            Err(LogError::NotThisFormat { .. })
389        ));
390        assert!(matches!(
391            unpack(b"PK\x03\x04 and nothing more"),
392            Err(LogError::Archive { .. })
393        ));
394        assert!(!is_archive(b"PK"));
395        assert!(is_archive(b"PK\x05\x06"));
396    }
397
398    #[test]
399    fn an_empty_archive_holds_no_files() {
400        let bytes = archive(&[]);
401        assert!(is_archive(&bytes));
402        assert_eq!(unpack(&bytes), Ok(Vec::new()));
403    }
404
405    proptest::proptest! {
406        /// Hostile bytes after a zip's opening are refused or unpacked within the budget, never a
407        /// panic.
408        #[test]
409        fn any_bytes_are_refused_or_held_to_the_budget(
410            tail in proptest::collection::vec(proptest::num::u8::ANY, 0..600),
411        ) {
412            let mut bytes = b"PK\x03\x04".to_vec();
413            bytes.extend(tail);
414            if let Ok(members) = unpack_within(&bytes, 4096) {
415                let total: usize = members.iter().map(|member| member.bytes.len()).sum();
416                proptest::prop_assert!(total <= 4096);
417            }
418        }
419
420        /// A real archive with bytes overwritten anywhere is refused or held to the budget.
421        #[test]
422        fn a_damaged_archive_is_refused_or_held_to_the_budget(
423            edits in proptest::collection::vec((0usize..400, proptest::num::u8::ANY), 1..8),
424        ) {
425            let mut bytes = archive(&[("a.csv", &[b'7'; 300][..]), ("b/c.txt", b"0.0\t1\t2F\t3")]);
426            for (at, value) in edits {
427                let at = at % bytes.len();
428                bytes[at] = value;
429            }
430            if let Ok(members) = unpack_within(&bytes, 4096) {
431                let total: usize = members.iter().map(|member| member.bytes.len()).sum();
432                proptest::prop_assert!(total <= 4096);
433            }
434        }
435    }
436
437    #[test]
438    fn a_long_name_is_shortened_in_errors() {
439        let name = "x".repeat(1000);
440        assert_eq!(shortened(&name).chars().count(), 61);
441        assert_eq!(shortened("short"), "short");
442    }
443}