tile_loader: Download tiles in separate threads

src/tile_loader.rs

@@ -8,8 +8,8 @@ use std::collections::hash_set::HashSet;
 use std::fs::File;
 use std::io::Write;
 use std::path::{Path, PathBuf};
+use std::sync::{Arc, mpsc, Mutex};
 use std::sync::mpsc::TryRecvError;
-use std::sync::mpsc;
 use std::thread;
 use tile::Tile;
 use tile_source::TileSource;
@@ -49,85 +49,154 @@ impl TileLoader {
     )
         where F: Fn(Tile) + Sync + Send + 'static,
     {
-        let mut client_opt = None;
-        let mut queue: Vec<(Tile, String, PathBuf, bool)> = vec![];
+        let mut queue: Vec<TileRequest> = vec![];
+        let remote_queue: Arc<Mutex<Vec<TileRequest>>> = Arc::new(Mutex::new(vec![]));
+        let mut view_opt: Option<View> = None;
+
+        let arc_notice_func = Arc::new(notice_func);
+
+        let (remote_request_tx, remote_request_rx) = mpsc::channel();
+        {
+            let arc_request_rx = Arc::new(Mutex::new(remote_request_rx));
+            for id in 0..2 {
+                let remote_queue = Arc::clone(&remote_queue);
+                let arc_request_rx = Arc::clone(&arc_request_rx);
+                let result_tx = result_tx.clone();
+                let arc_notice_func = Arc::clone(&arc_notice_func);
+                thread::spawn(move || Self::work_remote(id, remote_queue, arc_request_rx, result_tx, arc_notice_func));
+            }
+        }
 
         'outer: while let Ok(message) = request_rx.recv() {
-            let mut view_opt: Option<View> = None;
+            let mut need_to_sort = true;
 
             match message {
-                LoaderMessage::SetViewLocation{view} => {
+                LoaderMessage::SetView(view) => {
                     view_opt = Some(view);
                 },
-                LoaderMessage::GetTile{tile, url, path, write_to_file} => {
-                    queue.push((tile, url, path, write_to_file));
+                LoaderMessage::GetTile(request) => {
+                    queue.push(request);
                 }
             }
 
             loop {
                 loop {
-                    match request_rx.try_recv() {
-                        Ok(LoaderMessage::SetViewLocation{view}) => {
+                    let message = request_rx.try_recv();
+
+                    match message {
+                        Ok(LoaderMessage::SetView(view)) => {
                             view_opt = Some(view);
+                            need_to_sort = true;
                         },
-                        Ok(LoaderMessage::GetTile{tile, url, path, write_to_file}) => {
-                            queue.push((tile, url, path, write_to_file));
+                        Ok(LoaderMessage::GetTile(request)) => {
+                            queue.push(request);
+                            need_to_sort = true;
                         },
                         Err(TryRecvError::Empty) => break,
                         Err(TryRecvError::Disconnected) => break 'outer,
                     }
                 }
 
-                if let Some(view) = view_opt {
-                    //TODO sort queue
-                    queue.as_mut_slice().sort_by(|&(tile_a, _, _, _), &(tile_b, _, _, _)| {
-                        compare_tiles(tile_a, tile_b, view)
-                    });
+                if need_to_sort {
+                    if let Some(view) = view_opt {
+                        need_to_sort = false;
+
+                        queue.as_mut_slice().sort_by(|a, b| {
+                            compare_tiles(a.tile, b.tile, view)
+                        });
+
+                        if let Ok(mut remote_queue) = remote_queue.lock() {
+                            remote_queue.as_mut_slice().sort_by(|a, b| {
+                                compare_tiles(a.tile, b.tile, view)
+                            });
+                        }
+                    }
                 }
 
                 match queue.pop() {
                     None => break,
-                    Some((tile, url, path, write_to_file)) => {
-                        println!("queue {:?} {:?}", tile, path);
-                        match image::open(&path) {
+                    Some(request) => {
+                        match image::open(&request.path) {
                             Ok(img) => {
-                                result_tx.send((tile, Some(img))).unwrap();
-                                notice_func(tile);
+                                if let Err(_) = result_tx.send((request.tile, Some(img))) {
+                                    break 'outer;
+                                }
+                                arc_notice_func(request.tile);
                                 continue;
                             },
                             Err(_) => {
-                                //TODO do not try to create a client every time when it failed before
-                                if client_opt.is_none() {
-                                    client_opt = Client::builder().build().ok();
-                                }
-
-                                if let Some(ref client) = client_opt {
-                                    println!("use client {:?}", tile);
-                                    if let Ok(mut response) = client.get(&url).send() {
-                                        let mut buf: Vec<u8> = vec![];
-                                        response.copy_to(&mut buf).unwrap();
-                                        if let Ok(img) = image::load_from_memory(&buf) {
-                                            result_tx.send((tile, Some(img))).unwrap();
-                                            notice_func(tile);
-
-                                            if write_to_file {
-                                                //TODO do something on write errors
-                                                let _ = Self::write_to_file(&path, &buf);
-                                            }
-
-                                            continue;
-                                        }
+                                if let Ok(mut remote_queue) = remote_queue.lock() {
+                                    //TODO restrict size of remote_queue
+                                    remote_queue.push(request);
+                                    if let Some(view) = view_opt {
+                                        remote_queue.as_mut_slice().sort_by(|a, b| {
+                                            compare_tiles(a.tile, b.tile, view)
+                                        });
+                                    }
+                                    if remote_request_tx.send(RemoteLoaderMessage::PopQueue).is_err() {
+                                        //TODO remote worker terminated
                                     }
                                 }
                             },
                         }
-                        result_tx.send((tile, None)).unwrap();
                     },
                 }
             }
         }
     }
 
+    fn work_remote<F>(
+        thread_id: u32,
+        queue: Arc<Mutex<Vec<TileRequest>>>,
+        request_rx: Arc<Mutex<mpsc::Receiver<RemoteLoaderMessage>>>,
+        result_tx: mpsc::Sender<(Tile, Option<DynamicImage>)>,
+        notice_func: Arc<F>,
+    )
+        where F: Fn(Tile) + Sync + Send + 'static,
+    {
+        let mut client_opt = None;
+
+        loop {
+            let message = request_rx.lock().ok().and_then(|r| r.recv().ok());
+            match message {
+                None => break,
+                Some(RemoteLoaderMessage::Terminate) => break,
+                Some(RemoteLoaderMessage::PopQueue) => {
+                    let ele: Option<TileRequest> = queue.lock().ok().and_then(|mut q| q.pop());
+
+                    if let Some(request) = ele {
+                        println!("thread {}: queue {:?} {:?}", thread_id, request.tile, request.path);
+                        if client_opt.is_none() {
+                            client_opt = Client::builder().build().ok();
+                        }
+
+                        if let Some(Ok(mut response)) = client_opt.as_ref().map(|c| c.get(&request.url).send()) {
+                            let mut buf: Vec<u8> = vec![];
+                            response.copy_to(&mut buf).unwrap();
+                            if let Ok(img) = image::load_from_memory(&buf) {
+                                if result_tx.send((request.tile, Some(img))).is_err() {
+                                    break;
+                                }
+
+                                notice_func(request.tile);
+
+                                if request.write_to_file {
+                                    //TODO do something on write errors
+                                    let _ = Self::write_to_file(&request.path, &buf);
+                                }
+
+                                continue;
+                            }
+                        }
+                        if result_tx.send((request.tile, None)).is_err() {
+                            break;
+                        }
+                    }
+                },
+            }
+        }
+    }
+
     pub fn async_request(&mut self, tile_coord: TileCoord, source: &TileSource, write_to_file: bool) {
         if tile_coord.zoom > source.max_tile_zoom() {
             return;
@@ -137,12 +206,14 @@ impl TileLoader {
 
         if !self.pending.contains(&tile) {
             self.pending.insert(tile);
-            self.request_tx.send(LoaderMessage::GetTile{
-                tile: tile,
-                url: source.remote_tile_url(tile_coord),
-                path: source.local_tile_path(tile_coord),
-                write_to_file: write_to_file,
-            }).unwrap();
+            self.request_tx.send(LoaderMessage::GetTile(
+                TileRequest {
+                    tile: tile,
+                    url: source.remote_tile_url(tile_coord),
+                    path: source.local_tile_path(tile_coord),
+                    write_to_file: write_to_file,
+                }
+            )).unwrap();
         }
     }
 
@@ -172,7 +243,6 @@ impl TileLoader {
                 }
 
                 if let Some(ref client) = self.client {
-                    println!("use client {:?}", tile);
                     if let Ok(mut response) = client.get(&source.remote_tile_url(tile)).send() {
                         let mut buf: Vec<u8> = vec![];
                         response.copy_to(&mut buf).unwrap();
@@ -196,28 +266,37 @@ impl TileLoader {
     }
 
     pub fn set_view_location(&mut self, view: View) {
-        self.request_tx.send(LoaderMessage::SetViewLocation{
-            view: view,
-        }).unwrap();
+        self.request_tx.send(LoaderMessage::SetView(view)).unwrap();
     }
 
     fn write_to_file<P: AsRef<Path>>(path: P, img_data: &[u8]) -> ::std::io::Result<()> {
-
         if let Some(dir) = path.as_ref().parent() {
             ::std::fs::create_dir_all(dir)?;
         }
 
-        //TODO remove
-        println!("write file {:?}", path.as_ref());
-
         let mut file = File::create(path)?;
         file.write_all(img_data)
     }
 }
 
+#[derive(Debug)]
+struct TileRequest {
+    pub tile: Tile,
+    pub url: String,
+    pub path: PathBuf,
+    pub write_to_file: bool,
+}
+
+#[derive(Debug)]
 enum LoaderMessage {
-    GetTile{tile: Tile, url: String, path: PathBuf, write_to_file: bool},
-    SetViewLocation{view: View},
+    GetTile(TileRequest),
+    SetView(View),
+}
+
+#[derive(Debug)]
+enum RemoteLoaderMessage {
+    PopQueue,
+    Terminate,
 }
 
 fn compare_tiles(a: Tile, b: Tile, view: View) -> Ordering {