store known hosts in redis only after successful fetch

hunter/src/main.rs

@@ -42,10 +42,10 @@ async fn main() {
     systemd_status("Starting scheduler");
     let mut scheduler = scheduler::Scheduler::new();
     for host in config.hosts.into_iter() {
-        scheduler.introduce(&mut redis_man, host).await;
+        scheduler.introduce(host).await;
     }
     for host in crate::redis_store::get_hosts(&mut redis_man).await.into_iter() {
-        scheduler.introduce(&mut redis_man, host).await;
+        scheduler.introduce(host).await;
     }
 
     systemd_status("Starting HTTP client");
@@ -91,7 +91,7 @@ async fn main() {
                         }
                         Message::IntroduceHosts { hosts } => {
                             for host in hosts.into_iter() {
-                                scheduler.introduce(&mut redis_man, host).await;
+                                scheduler.introduce(host).await;
                             }
                         }
                     }

hunter/src/scheduler.rs

@@ -30,13 +30,10 @@ impl Scheduler {
         self.queue.len()
     }
 
-    pub async fn introduce(&mut self, redis_man: &mut redis::aio::ConnectionManager, host: String) {
+    pub async fn introduce(&mut self, host: String) {
         let now = Instant::now();
 
         if self.instances.get(&host).is_none() {
-            // save for later
-            crate::redis_store::save_host(redis_man, &host).await;
-
             self.instances.insert(host.clone(), Instance {
                 last_fetch: None,
                 no_updates: 0,

hunter/src/worker.rs

@@ -68,6 +68,9 @@ pub fn fetch_and_process(
                     .collect();
                 let _ = message_tx.send(Message::IntroduceHosts { hosts });
 
+                // successfully fetched, save for future run
+                crate::redis_store::save_host(&mut redis_man, &host).await;
+
                 message_tx.send(Message::Fetched {
                     host: host.clone(),
                     new_posts,