|
|
@@ -1,8 +1,10 @@
|
|
1
|
|
-use osmpbf::{Element, ElementReader};
|
|
|
1
|
+use scoped_threadpool::Pool;
|
|
|
2
|
+use coord::LatLon;
|
|
|
3
|
+use osmpbf::{Blob, BlobDecode, BlobReader};
|
|
2
|
4
|
use regex::Regex;
|
|
3
|
5
|
use std::path::{Path, PathBuf};
|
|
|
6
|
+use std::sync::mpsc::sync_channel;
|
|
4
|
7
|
use std::thread;
|
|
5
|
|
-use coord::LatLon;
|
|
6
|
8
|
|
|
7
|
9
|
|
|
8
|
10
|
#[derive(Debug, Eq, PartialEq)]
|
|
|
@@ -21,50 +23,152 @@ impl<T, E> From<Result<T, E>> for ControlFlow
|
|
21
|
23
|
}
|
|
22
|
24
|
}
|
|
23
|
25
|
|
|
24
|
|
-//TODO Add callbacks for other events: search finished, on error, ...
|
|
25
|
|
-pub fn search_pbf<P, F>(
|
|
|
26
|
+enum WorkerMessage {
|
|
|
27
|
+ PleaseStop,
|
|
|
28
|
+ DoBlob(Blob),
|
|
|
29
|
+}
|
|
|
30
|
+
|
|
|
31
|
+pub fn par_search<P, F, G>(
|
|
26
|
32
|
pbf_path: P,
|
|
27
|
33
|
search_pattern: &str,
|
|
28
|
|
- update_func: F,
|
|
|
34
|
+ found_func: F,
|
|
|
35
|
+ finished_func: G,
|
|
29
|
36
|
) -> Result<thread::JoinHandle<()>, String>
|
|
30
|
37
|
where P: AsRef<Path>,
|
|
31
|
|
- F: Fn(LatLon) -> ControlFlow + Send + 'static,
|
|
|
38
|
+ F: Fn(Vec<LatLon>) -> ControlFlow + Send + 'static,
|
|
|
39
|
+ G: Fn(Result<(), String>) + Send + 'static,
|
|
32
|
40
|
{
|
|
33
|
|
- let pathbuf = PathBuf::from(pbf_path.as_ref());
|
|
34
|
|
- let re = Regex::new(search_pattern)
|
|
35
|
|
- .map_err(|e| format!("{}", e))?;
|
|
36
|
|
- let reader = ElementReader::from_path(&pathbuf)
|
|
37
|
|
- .map_err(|e| format!("Failed to read PBF file {:?}: {}", pbf_path.as_ref(), e))?;
|
|
38
|
|
-
|
|
|
41
|
+ let pbf_path = PathBuf::from(pbf_path.as_ref());
|
|
|
42
|
+ let search_pattern = search_pattern.to_string();
|
|
39
|
43
|
let handle = thread::spawn(move|| {
|
|
40
|
|
- reader.for_each(|element| {
|
|
41
|
|
- match element {
|
|
42
|
|
- Element::Node(node) => {
|
|
43
|
|
- for (_key, val) in node.tags() {
|
|
44
|
|
- if re.is_match(val) {
|
|
45
|
|
- let pos = LatLon::new(node.lat(), node.lon());
|
|
46
|
|
- if update_func(pos) == ControlFlow::Break {
|
|
47
|
|
- return;
|
|
|
44
|
+ let res = par_search_blocking(pbf_path, &search_pattern, found_func);
|
|
|
45
|
+ finished_func(res);
|
|
|
46
|
+ });
|
|
|
47
|
+
|
|
|
48
|
+ Ok(handle)
|
|
|
49
|
+}
|
|
|
50
|
+
|
|
|
51
|
+pub fn par_search_blocking<P, F>(
|
|
|
52
|
+ pbf_path: P,
|
|
|
53
|
+ search_pattern: &str,
|
|
|
54
|
+ found_func: F,
|
|
|
55
|
+) -> Result<(), String>
|
|
|
56
|
+where P: AsRef<Path>,
|
|
|
57
|
+ F: Fn(Vec<LatLon>) -> ControlFlow + Send + 'static,
|
|
|
58
|
+{
|
|
|
59
|
+ let num_threads = ::num_cpus::get();
|
|
|
60
|
+ let mut pool = Pool::new(num_threads as u32);
|
|
|
61
|
+
|
|
|
62
|
+ pool.scoped(|scope| {
|
|
|
63
|
+ let re = Regex::new(search_pattern)
|
|
|
64
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
65
|
+ let mut reader = BlobReader::from_path(&pbf_path)
|
|
|
66
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
67
|
+
|
|
|
68
|
+ let mut chans = Vec::with_capacity(num_threads);
|
|
|
69
|
+ let (result_tx, result_rx) = sync_channel::<(usize, Result<Vec<LatLon>, String>)>(0);
|
|
|
70
|
+
|
|
|
71
|
+ for thread_id in 0..num_threads {
|
|
|
72
|
+ let re = re.clone();
|
|
|
73
|
+ let result_tx = result_tx.clone();
|
|
|
74
|
+
|
|
|
75
|
+ let (request_tx, request_rx) = sync_channel::<WorkerMessage>(0);
|
|
|
76
|
+ chans.push(request_tx);
|
|
|
77
|
+
|
|
|
78
|
+ scope.execute(move || {
|
|
|
79
|
+ for request in request_rx.iter() {
|
|
|
80
|
+ match request {
|
|
|
81
|
+ WorkerMessage::PleaseStop => return,
|
|
|
82
|
+ WorkerMessage::DoBlob(blob) => {
|
|
|
83
|
+ let mut matches = vec![];
|
|
|
84
|
+ let block = match blob.decode() {
|
|
|
85
|
+ Ok(b) => b,
|
|
|
86
|
+ Err(err) => {
|
|
|
87
|
+ let _ = result_tx.send((thread_id, Err(format!("{}", err))));
|
|
|
88
|
+ return;
|
|
|
89
|
+ }
|
|
|
90
|
+ };
|
|
|
91
|
+ if let BlobDecode::OsmData(block) = block {
|
|
|
92
|
+ for node in block.groups().flat_map(|g| g.nodes()) {
|
|
|
93
|
+ for (_key, val) in node.tags() {
|
|
|
94
|
+ if re.is_match(val) {
|
|
|
95
|
+ let pos = LatLon::new(node.lat(), node.lon());
|
|
|
96
|
+ matches.push(pos);
|
|
|
97
|
+ break;
|
|
|
98
|
+ }
|
|
|
99
|
+ }
|
|
|
100
|
+ }
|
|
|
101
|
+
|
|
|
102
|
+ for node in block.groups().flat_map(|g| g.dense_nodes()) {
|
|
|
103
|
+ for (_key, val) in node.tags() {
|
|
|
104
|
+ if re.is_match(val) {
|
|
|
105
|
+ let pos = LatLon::new(node.lat(), node.lon());
|
|
|
106
|
+ matches.push(pos);
|
|
|
107
|
+ break;
|
|
|
108
|
+ }
|
|
|
109
|
+ }
|
|
|
110
|
+ }
|
|
48
|
111
|
}
|
|
49
|
|
- break;
|
|
50
|
|
- }
|
|
51
|
|
- }
|
|
52
|
|
- },
|
|
53
|
|
- Element::DenseNode(node) => {
|
|
54
|
|
- for (_key, val) in node.tags() {
|
|
55
|
|
- if re.is_match(val) {
|
|
56
|
|
- let pos = LatLon::new(node.lat(), node.lon());
|
|
57
|
|
- if update_func(pos) == ControlFlow::Break {
|
|
|
112
|
+ if result_tx.send((thread_id, Ok(matches))).is_err() {
|
|
58
|
113
|
return;
|
|
59
|
114
|
}
|
|
60
|
|
- break;
|
|
61
|
115
|
}
|
|
62
|
|
- }
|
|
|
116
|
+ };
|
|
|
117
|
+ }
|
|
|
118
|
+ });
|
|
|
119
|
+ }
|
|
|
120
|
+
|
|
|
121
|
+ let mut stopped_threads = 0;
|
|
|
122
|
+
|
|
|
123
|
+ // send initial message to each worker thread
|
|
|
124
|
+ for thread_id in 0..num_threads {
|
|
|
125
|
+ match reader.next() {
|
|
|
126
|
+ Some(Ok(blob)) => {
|
|
|
127
|
+ chans[thread_id].send(WorkerMessage::DoBlob(blob))
|
|
|
128
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
129
|
+
|
|
|
130
|
+ },
|
|
|
131
|
+ Some(Err(err)) => {
|
|
|
132
|
+ return Err(format!("{}", err));
|
|
|
133
|
+ },
|
|
|
134
|
+ None => {
|
|
|
135
|
+ chans[thread_id].send(WorkerMessage::PleaseStop)
|
|
|
136
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
137
|
+ stopped_threads += 1;
|
|
63
|
138
|
},
|
|
64
|
|
- _ => {},
|
|
65
|
139
|
}
|
|
66
|
|
- }).unwrap();
|
|
67
|
|
- });
|
|
|
140
|
+ }
|
|
68
|
141
|
|
|
69
|
|
- Ok(handle)
|
|
|
142
|
+ if stopped_threads == num_threads {
|
|
|
143
|
+ return Ok(());
|
|
|
144
|
+ }
|
|
|
145
|
+
|
|
|
146
|
+ for (thread_id, matches) in result_rx.iter() {
|
|
|
147
|
+ let matches = matches?;
|
|
|
148
|
+
|
|
|
149
|
+ if found_func(matches) == ControlFlow::Break {
|
|
|
150
|
+ break;
|
|
|
151
|
+ }
|
|
|
152
|
+
|
|
|
153
|
+ match reader.next() {
|
|
|
154
|
+ Some(Ok(blob)) => {
|
|
|
155
|
+ chans[thread_id].send(WorkerMessage::DoBlob(blob))
|
|
|
156
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
157
|
+ },
|
|
|
158
|
+ Some(Err(err)) => {
|
|
|
159
|
+ return Err(format!("{}", err));
|
|
|
160
|
+ },
|
|
|
161
|
+ None => {
|
|
|
162
|
+ chans[thread_id].send(WorkerMessage::PleaseStop)
|
|
|
163
|
+ .map_err(|e| format!("{}", e))?;
|
|
|
164
|
+ stopped_threads += 1;
|
|
|
165
|
+ if stopped_threads == num_threads {
|
|
|
166
|
+ break;
|
|
|
167
|
+ }
|
|
|
168
|
+ }
|
|
|
169
|
+ }
|
|
|
170
|
+ }
|
|
|
171
|
+
|
|
|
172
|
+ Ok(())
|
|
|
173
|
+ })
|
|
70
|
174
|
}
|
