This adds at least one memory allocation (`collect` into a `Vec`) but is still faster for a reasonable number of IDs.

examples/indexed.rs

@@ -0,0 +1,39 @@
+// Count the number of buildings and their nodes from a PBF file
+// given as the first command line argument.
+
+extern crate osmpbf;
+
+use osmpbf::{IndexedReader, Element};
+use std::error::Error;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // Read command line argument and create IndexedReader
+    let arg = std::env::args_os()
+        .nth(1)
+        .ok_or("need a *.osm.pbf file as argument")?;
+    let mut reader = IndexedReader::from_path(&arg)?;
+
+    println!("Counting...");
+    let mut ways = 0;
+    let mut nodes = 0;
+
+    reader.read_ways_and_deps(
+        |way| {
+            // Filter ways. Return true if tags contain "building": "yes".
+            way.tags().any(|key_value| key_value == ("building", "yes"))
+        },
+        |element| {
+            // Increment counter for ways and nodes
+            match element {
+                Element::Way(_way) => ways += 1,
+                Element::Node(_node) => nodes += 1,
+                Element::DenseNode(_dense_node) => nodes += 1,
+                Element::Relation(_) => {}, // should not occur
+            }
+        },
+    )?;
+
+    // Print result
+    println!("ways:  {}\nnodes: {}", ways, nodes);
+    Ok(())
+}

src/blob.rs

@@ -121,6 +121,35 @@ impl Blob {
     }
 }
 
+/// A blob header.
+///
+/// Just contains information about the size and type of the following `Blob`.
+#[derive(Clone, Debug)]
+pub struct BlobHeader {
+    header: fileformat::BlobHeader,
+}
+
+impl BlobHeader {
+    fn new(header: fileformat::BlobHeader) -> Self {
+        BlobHeader { header }
+    }
+
+    /// Returns the type of the following blob.
+    pub fn blob_type(&self) -> BlobType {
+        match self.header.get_field_type() {
+            "OSMHeader" => BlobType::OsmHeader,
+            "OSMData" => BlobType::OsmData,
+            x => BlobType::Unknown(x),
+        }
+    }
+
+    /// Returns the size of the following blob in bytes.
+    pub fn get_blob_size(&self) -> i32 {
+        self.header.get_datasize()
+    }
+}
+
+
 /// A reader for PBF files that allows iterating over `Blob`s.
 #[derive(Clone, Debug)]
 pub struct BlobReader<R: Read> {
@@ -154,6 +183,51 @@ impl<R: Read> BlobReader<R> {
             last_blob_ok: true,
         }
     }
+
+    fn read_blob_header(&mut self) -> Option<Result<fileformat::BlobHeader>> {
+        let header_size: u64 = match self.reader.read_u32::<byteorder::BigEndian>() {
+            Ok(n) => {
+                self.offset = self.offset.map(|x| ByteOffset(x.0 + 4));
+                u64::from(n)
+            }
+            Err(e) => {
+                self.offset = None;
+                match e.kind() {
+                    ::std::io::ErrorKind::UnexpectedEof => {
+                        //TODO This also accepts corrupted files in the case of 1-3 available bytes
+                        return None;
+                    }
+                    _ => {
+                        self.last_blob_ok = false;
+                        return Some(Err(new_blob_error(BlobError::InvalidHeaderSize)));
+                    }
+                }
+            }
+        };
+
+        if header_size >= MAX_BLOB_HEADER_SIZE {
+            self.last_blob_ok = false;
+            return Some(Err(new_blob_error(BlobError::HeaderTooBig {
+                size: header_size,
+            })));
+        }
+
+        let header: fileformat::BlobHeader =
+            match parse_message_from_reader(&mut self.reader.by_ref().take(header_size)) {
+                Ok(header) => header,
+                Err(e) => {
+                    self.offset = None;
+                    self.last_blob_ok = false;
+                    return Some(Err(new_protobuf_error(e, "blob header")));
+                }
+            };
+
+        self.offset = self
+            .offset
+            .map(|x| ByteOffset(x.0 + header_size));
+
+        Some(Ok(header))
+    }
 }
 
 impl BlobReader<BufReader<File>> {
@@ -195,43 +269,12 @@ impl<R: Read> Iterator for BlobReader<R> {
 
         let prev_offset = self.offset;
 
-        let header_size: u64 = match self.reader.read_u32::<byteorder::BigEndian>() {
-            Ok(n) => {
-                self.offset = self.offset.map(|x| ByteOffset(x.0 + 4));
-                u64::from(n)
-            }
-            Err(e) => {
-                self.offset = None;
-                match e.kind() {
-                    ::std::io::ErrorKind::UnexpectedEof => {
-                        //TODO This also accepts corrupted files in the case of 1-3 available bytes
-                        return None;
-                    }
-                    _ => {
-                        self.last_blob_ok = false;
-                        return Some(Err(new_blob_error(BlobError::InvalidHeaderSize)));
-                    }
-                }
-            }
+        let header = match self.read_blob_header() {
+            Some(Ok(header)) => header,
+            Some(Err(err)) => return Some(Err(err)),
+            None => return None,
         };
 
-        if header_size >= MAX_BLOB_HEADER_SIZE {
-            self.last_blob_ok = false;
-            return Some(Err(new_blob_error(BlobError::HeaderTooBig {
-                size: header_size,
-            })));
-        }
-
-        let header: fileformat::BlobHeader =
-            match parse_message_from_reader(&mut self.reader.by_ref().take(header_size)) {
-                Ok(header) => header,
-                Err(e) => {
-                    self.offset = None;
-                    self.last_blob_ok = false;
-                    return Some(Err(new_protobuf_error(e, "blob header")));
-                }
-            };
-
         let blob: fileformat::Blob = match parse_message_from_reader(
             &mut self.reader.by_ref().take(header.get_datasize() as u64),
         ) {
@@ -245,7 +288,7 @@ impl<R: Read> Iterator for BlobReader<R> {
 
         self.offset = self
             .offset
-            .map(|x| ByteOffset(x.0 + header_size + header.get_datasize() as u64));
+            .map(|x| ByteOffset(x.0 + header.get_datasize() as u64));
 
         Some(Ok(Blob::new(header, blob, prev_offset)))
     }
@@ -326,6 +369,34 @@ impl<R: Read + Seek> BlobReader<R> {
             }
         }
     }
+
+    /// Read and return next `BlobHeader` but skip the following `Blob`. This allows really fast
+    /// iteration of the PBF structure if only the byte offset and `BlobType` are important.
+    /// On success, returns the `BlobHeader` and the byte offset of the header which can also be
+    /// used as an offset for reading the entire `Blob` (including header).
+    pub fn next_header_skip_blob(&mut self) -> Option<Result<(BlobHeader, Option<ByteOffset>)>> {
+        // Stop iteration if there was an error.
+        if !self.last_blob_ok {
+            return None;
+        }
+
+        let prev_offset = self.offset;
+
+        // read header
+        let header = match self.read_blob_header() {
+            Some(Ok(header)) => header,
+            Some(Err(err)) => return Some(Err(err)),
+            None => return None,
+        };
+
+        // skip blob (which also adjusts self.offset)
+        if let Err(err) = self.seek_raw(SeekFrom::Current(header.get_datasize() as i64)) {
+            self.last_blob_ok = false;
+            return Some(Err(err));
+        }
+
+        Some(Ok((BlobHeader::new(header), prev_offset)))
+    }
 }
 
 impl BlobReader<BufReader<File>> {

src/indexed.rs

@@ -0,0 +1,286 @@
+//! Speed up searches by using an index
+
+use error::Result;
+use std::collections::BTreeSet;
+use std::fs::File;
+use std::io::{Read, Seek};
+use std::ops::RangeInclusive;
+use std::path::Path;
+use {BlobReader, BlobType, ByteOffset, Element, Way};
+
+/// Stores the minimum and maximum id of every element type.
+#[derive(Debug)]
+pub struct IdRanges {
+    node_ids: Option<RangeInclusive<i64>>,
+    way_ids: Option<RangeInclusive<i64>>,
+    relation_ids: Option<RangeInclusive<i64>>,
+}
+
+/// Returns true if the given set contains at least one value that is inside the given range.
+fn range_included(range: RangeInclusive<i64>, node_ids: &BTreeSet<i64>) -> bool {
+    node_ids.range(range).next().is_some()
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum SimpleBlobType {
+    Header,
+    Primitive,
+    Unknown,
+}
+
+#[derive(Debug)]
+struct BlobInfo {
+    offset: ByteOffset,
+    blob_type: SimpleBlobType,
+    id_ranges: Option<IdRanges>,
+}
+
+/// Allows filtering elements and iterating over their dependencies.
+/// It chooses an efficient method for navigating the PBF structure to achieve this in reasonable
+/// time and with reasonable memory.
+pub struct IndexedReader<R: Read + Seek> {
+    reader: BlobReader<R>,
+    index: Vec<BlobInfo>,
+}
+
+impl<R: Read + Seek> IndexedReader<R> {
+    /// Creates a new `IndexedReader`.
+    ///
+    /// # Example
+    /// ```
+    /// use osmpbf::*;
+    ///
+    /// # fn foo() -> Result<()> {
+    /// let f = std::fs::File::open("tests/test.osm.pbf")?;
+    /// let buf_reader = std::io::BufReader::new(f);
+    ///
+    /// let reader = IndexedReader::new(buf_reader)?;
+    ///
+    /// # Ok(())
+    /// # }
+    /// # foo().unwrap();
+    /// ```
+    pub fn new(reader: R) -> Result<Self> {
+        let reader = BlobReader::new_seekable(reader)?;
+        Ok(Self {
+            reader,
+            index: vec![],
+        })
+    }
+
+    pub fn create_index(&mut self) -> Result<()> {
+        // remove old items
+        self.index.clear();
+
+        while let Some(result) = self.reader.next_header_skip_blob() {
+            let (header, offset) = result?;
+            // Reader is seekable, so offset should be Some(ByteOffset)
+            let offset = offset.unwrap();
+            let blob_type = match header.blob_type() {
+                BlobType::OsmHeader => SimpleBlobType::Header,
+                BlobType::OsmData => SimpleBlobType::Primitive,
+                BlobType::Unknown(_) => SimpleBlobType::Unknown,
+            };
+
+            self.index.push(BlobInfo {
+                offset,
+                blob_type,
+                id_ranges: None,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Filter ways using a closure and return matching ways and their dependent nodes (`Node`s and
+    /// `DenseNode`s) in another closure.
+    ///
+    /// # Example
+    /// ```
+    /// use osmpbf::*;
+    ///
+    /// # fn foo() -> Result<()> {
+    /// let mut reader = IndexedReader::from_path("tests/test.osm.pbf")?;
+    /// let mut ways = 0;
+    /// let mut nodes = 0;
+    ///
+    /// // Filter all ways that are buildings and count their nodes.
+    /// reader.read_ways_and_deps(
+    ///     |way| {
+    ///         // Filter ways. Return true if tags contain "building": "yes".
+    ///         way.tags().any(|key_value| key_value == ("building", "yes"))
+    ///     },
+    ///     |element| {
+    ///         // Increment counter
+    ///         match element {
+    ///             Element::Way(way) => ways += 1,
+    ///             Element::Node(node) => nodes += 1,
+    ///             Element::DenseNode(dense_node) => nodes += 1,
+    ///             Element::Relation(_) => (), // should not occur
+    ///         }
+    ///     },
+    /// )?;
+    ///
+    /// println!("ways:  {}\nnodes: {}", ways, nodes);
+    ///
+    /// # assert_eq!(ways, 1);
+    /// # assert_eq!(nodes, 3);
+    /// # Ok(())
+    /// # }
+    /// # foo().unwrap();
+    /// ```
+    pub fn read_ways_and_deps<F, E>(
+        &mut self,
+        mut filter: F,
+        mut element_callback: E,
+    ) -> Result<()>
+    where
+        F: for<'a> FnMut(&Way<'a>) -> bool,
+        E: for<'a> FnMut(&Element<'a>),
+    {
+        // Create index
+        if self.index.is_empty() {
+            self.create_index()?;
+        }
+
+        let mut node_ids: BTreeSet<i64> = BTreeSet::new();
+
+        // First pass:
+        //   * Filter ways and store their dependencies as node IDs
+        //   * Store range of node IDs (min and max value) of each block
+        for info in &mut self.index {
+            //TODO do something useful with header blocks
+            if info.blob_type == SimpleBlobType::Primitive {
+                self.reader.seek(info.offset)?;
+                let blob = self.reader.next().ok_or_else(|| {
+                    ::std::io::Error::new(
+                        ::std::io::ErrorKind::UnexpectedEof,
+                        "could not read next blob",
+                    )
+                })??;
+                let block = blob.to_primitiveblock()?;
+                let mut min_node_id: Option<i64> = None;
+                let mut max_node_id: Option<i64> = None;
+                for group in block.groups() {
+                    // filter ways and record node IDs
+                    for way in group.ways() {
+                        if filter(&way) {
+                            let refs = way.refs();
+
+                            node_ids.extend(refs);
+
+                            // Return way
+                            element_callback(&Element::Way(way));
+                        }
+                    }
+
+                    // Check node IDs of this block, record min and max
+
+                    let mut check_min_max = |id| {
+                        min_node_id = Some(min_node_id.map_or(id, |x| x.min(id)));
+                        max_node_id = Some(max_node_id.map_or(id, |x| x.max(id)));
+                    };
+
+                    for node in group.nodes() {
+                        check_min_max(node.id())
+                    }
+                    for node in group.dense_nodes() {
+                        check_min_max(node.id)
+                    }
+                }
+                if let (Some(min), Some(max)) = (min_node_id, max_node_id) {
+                    info.id_ranges = Some(IdRanges {
+                        node_ids: Some(RangeInclusive::new(min, max)),
+                        way_ids: None,
+                        relation_ids: None,
+                    });
+                }
+            }
+        }
+
+        // Second pass:
+        //   * Iterate only over blobs that may include the node IDs we're searching for
+        for info in &mut self.index {
+            if info.blob_type == SimpleBlobType::Primitive {
+                if let Some(node_id_range) = info.id_ranges.as_ref().and_then(|r| r.node_ids.as_ref()) {
+                    if range_included(node_id_range.clone(), &node_ids) {
+                        //TODO Only collect into Vec if range has a reasonable size
+                        let node_ids: Vec<i64> = node_ids.range(node_id_range.clone()).map(|x| *x).collect();
+                        self.reader.seek(info.offset)?;
+                        let blob = self.reader.next().ok_or_else(|| {
+                            ::std::io::Error::new(
+                                ::std::io::ErrorKind::UnexpectedEof,
+                                "could not read next blob",
+                            )
+                        })??;
+                        let block = blob.to_primitiveblock()?;
+                        for group in block.groups() {
+                            for node in group.nodes() {
+                                if node_ids.binary_search(&node.id()).is_ok() {
+                                    // ID found, return node
+                                    element_callback(&Element::Node(node));
+                                }
+                            }
+                            for node in group.dense_nodes() {
+                                if node_ids.binary_search(&node.id).is_ok() {
+                                    // ID found, return dense node
+                                    element_callback(&Element::DenseNode(node));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl IndexedReader<File> {
+    /// Creates a new `IndexedReader` from a given path.
+    ///
+    /// # Example
+    /// ```
+    /// use osmpbf::*;
+    ///
+    /// # fn foo() -> Result<()> {
+    /// let reader = IndexedReader::from_path("tests/test.osm.pbf")?;
+    ///
+    /// # Ok(())
+    /// # }
+    /// # foo().unwrap();
+    /// ```
+    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
+        //TODO take some more measurements to determine if `BufReader` should be used here
+        let f = File::open(path)?;
+        Self::new(f)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_range_included_set() {
+        let mut set = BTreeSet::<i64>::new();
+        set.extend(&[1,2,6]);
+
+        assert_eq!(range_included(RangeInclusive::new(0, 0), &set), false);
+        assert_eq!(range_included(RangeInclusive::new(1, 1), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(2, 2), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(3, 3), &set), false);
+        assert_eq!(range_included(RangeInclusive::new(3, 5), &set), false);
+        assert_eq!(range_included(RangeInclusive::new(3, 6), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(6, 6), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(7, 7), &set), false);
+        assert_eq!(range_included(RangeInclusive::new(0, 1), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(6, 7), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(2, 3), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(5, 6), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(5, 8), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(0, 8), &set), true);
+        assert_eq!(range_included(RangeInclusive::new(0, 4), &set), true);
+    }
+}

src/lib.rs

@@ -81,6 +81,7 @@ pub use block::*;
 pub use dense::*;
 pub use elements::*;
 pub use error::{BlobError, Error, ErrorKind, Result};
+pub use indexed::*;
 pub use mmap_blob::*;
 pub use reader::*;
 
@@ -89,6 +90,7 @@ pub mod block;
 pub mod dense;
 pub mod elements;
 mod error;
+pub mod indexed;
 pub mod mmap_blob;
 mod proto;
 pub mod reader;

tests/read.rs

@@ -198,3 +198,32 @@ fn par_read_elements() {
         assert_eq!(elements, 5);
     }
 }
+
+#[test]
+fn read_ways_and_deps() {
+    for path in &TEST_FILE_PATHS {
+        let mut reader = IndexedReader::from_path(path).unwrap();
+
+        let mut ways = 0;
+        let mut nodes = 0;
+
+        reader.read_ways_and_deps(
+            |way| {
+                way.tags()
+                   .find(|&key_value| key_value == ("building", "yes"))
+                   .is_some()
+            },
+            |element| {
+                match element {
+                    Element::Way(_) => ways += 1,
+                    Element::Node(_) => nodes += 1,
+                    Element::DenseNode(_) => nodes += 1,
+                    Element::Relation(_) => panic!(), // should not occur
+                }
+            },
+        ).unwrap();
+
+        assert_eq!(ways, 1);
+        assert_eq!(nodes, 3);
+    }
+}