2 Incheckningar

Upphovsman SHA1 Meddelande Datum
  Johannes Hofmann f1ebefa045 Merge branch 'substack-non-buffering' 5 år sedan
  substack bec900d97e changes to not buffer results in memory while preserving existing api 5 år sedan
3 ändrade filer med 13 tillägg och 17 borttagningar
  1. 4
    4
      src/blob.rs
  2. 2
    2
      src/indexed.rs
  3. 7
    11
      src/reader.rs

+ 4
- 4
src/blob.rs Visa fil

@@ -152,14 +152,14 @@ impl BlobHeader {
152 152
 
153 153
 /// A reader for PBF files that allows iterating over `Blob`s.
154 154
 #[derive(Clone, Debug)]
155
-pub struct BlobReader<R: Read> {
155
+pub struct BlobReader<R: Read + Send> {
156 156
     reader: R,
157 157
     /// Current reader offset in bytes from the start of the stream.
158 158
     offset: Option<ByteOffset>,
159 159
     last_blob_ok: bool,
160 160
 }
161 161
 
162
-impl<R: Read> BlobReader<R> {
162
+impl<R: Read + Send> BlobReader<R> {
163 163
     /// Creates a new `BlobReader`.
164 164
     ///
165 165
     /// # Example
@@ -258,7 +258,7 @@ impl BlobReader<BufReader<File>> {
258 258
     }
259 259
 }
260 260
 
261
-impl<R: Read> Iterator for BlobReader<R> {
261
+impl<R: Read + Send> Iterator for BlobReader<R> {
262 262
     type Item = Result<Blob>;
263 263
 
264 264
     fn next(&mut self) -> Option<Self::Item> {
@@ -294,7 +294,7 @@ impl<R: Read> Iterator for BlobReader<R> {
294 294
     }
295 295
 }
296 296
 
297
-impl<R: Read + Seek> BlobReader<R> {
297
+impl<R: Read + Seek + Send> BlobReader<R> {
298 298
     /// Creates a new `BlobReader` from the given reader that is seekable and will be initialized
299 299
     /// with a valid offset.
300 300
     ///

+ 2
- 2
src/indexed.rs Visa fil

@@ -38,12 +38,12 @@ struct BlobInfo {
38 38
 /// Allows filtering elements and iterating over their dependencies.
39 39
 /// It chooses an efficient method for navigating the PBF structure to achieve this in reasonable
40 40
 /// time and with reasonable memory.
41
-pub struct IndexedReader<R: Read + Seek> {
41
+pub struct IndexedReader<R: Read + Seek + Send> {
42 42
     reader: BlobReader<R>,
43 43
     index: Vec<BlobInfo>,
44 44
 }
45 45
 
46
-impl<R: Read + Seek> IndexedReader<R> {
46
+impl<R: Read + Seek + Send> IndexedReader<R> {
47 47
     /// Creates a new `IndexedReader`.
48 48
     ///
49 49
     /// # Example

+ 7
- 11
src/reader.rs Visa fil

@@ -10,11 +10,11 @@ use std::path::Path;
10 10
 
11 11
 /// A reader for PBF files that gives access to the stored elements: nodes, ways and relations.
12 12
 #[derive(Clone, Debug)]
13
-pub struct ElementReader<R: Read> {
13
+pub struct ElementReader<R: Read + Send> {
14 14
     blob_iter: BlobReader<R>,
15 15
 }
16 16
 
17
-impl<R: Read> ElementReader<R> {
17
+impl<R: Read + Send> ElementReader<R> {
18 18
     /// Creates a new `ElementReader`.
19 19
     ///
20 20
     /// # Example
@@ -68,11 +68,9 @@ impl<R: Read> ElementReader<R> {
68 68
     where
69 69
         F: for<'a> FnMut(Element<'a>),
70 70
     {
71
-        let blobs = self.blob_iter.collect::<Result<Vec<_>>>()?;
72
-
73 71
         //TODO do something useful with header blocks
74
-        for blob in &blobs {
75
-            match blob.decode() {
72
+        for blob in self.blob_iter {
73
+            match blob?.decode() {
76 74
                 Ok(BlobDecode::OsmHeader(_)) | Ok(BlobDecode::Unknown(_)) => {}
77 75
                 Ok(BlobDecode::OsmData(block)) => {
78 76
                     block.for_each_element(&mut f);
@@ -125,11 +123,9 @@ impl<R: Read> ElementReader<R> {
125 123
         ID: Fn() -> T + Sync + Send,
126 124
         T: Send,
127 125
     {
128
-        let blobs = self.blob_iter.collect::<Result<Vec<_>>>()?;
129
-
130
-        blobs
131
-            .into_par_iter()
132
-            .map(|blob| match blob.decode() {
126
+        self.blob_iter
127
+            .par_bridge()
128
+            .map(|blob| match blob?.decode() {
133 129
                 Ok(BlobDecode::OsmHeader(_)) | Ok(BlobDecode::Unknown(_)) => Ok(identity()),
134 130
                 Ok(BlobDecode::OsmData(block)) => Ok(block
135 131
                     .elements()