From 44a3d6820180af5b3ce2a183280a554e9a3b95ee Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Sat, 5 Nov 2022 03:25:30 +0100
Subject: [PATCH] hunter/worker: simplify error case, hardcode
 interval_after_error

---
 hunter/config.yaml   |  3 +--
 hunter/src/config.rs |  1 -
 hunter/src/main.rs   |  4 ----
 hunter/src/worker.rs | 13 ++++++++++---
 nixos-module.nix     |  1 -
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/hunter/config.yaml b/hunter/config.yaml
index 1cf88ce..1263279 100644
--- a/hunter/config.yaml
+++ b/hunter/config.yaml
@@ -26,5 +26,4 @@ hosts:
   - ubuntu.social
   - uwu.social
 
-interval_after_error: 7200
-max_workers: 128
+max_workers: 16
diff --git a/hunter/src/config.rs b/hunter/src/config.rs
index d70d617..6bb076b 100644
--- a/hunter/src/config.rs
+++ b/hunter/src/config.rs
@@ -2,7 +2,6 @@
 pub struct Config {
     pub redis: String,
     pub hosts: Vec<String>,
-    pub interval_after_error: u64,
     pub max_workers: usize,
 }
 
diff --git a/hunter/src/main.rs b/hunter/src/main.rs
index 7447c0f..2ddfe91 100644
--- a/hunter/src/main.rs
+++ b/hunter/src/main.rs
@@ -85,10 +85,6 @@ async fn main() {
                             workers_active -= 1;
                             scheduler.enqueue(host, new_posts > 0, next_interval);
                         }
-                        Message::Error { host } => {
-                            workers_active -= 1;
-                            scheduler.enqueue(host, false, Duration::from_secs(config.interval_after_error));
-                        }
                         Message::IntroduceHosts { hosts } => {
                             for host in hosts.into_iter() {
                                 scheduler.introduce(host).await;
diff --git a/hunter/src/worker.rs b/hunter/src/worker.rs
index ce6a653..56429a2 100644
--- a/hunter/src/worker.rs
+++ b/hunter/src/worker.rs
@@ -2,8 +2,12 @@ use std::collections::HashSet;
 use std::time::Duration;
 use crate::feed::Feed;
 
-const DEFAULT_INTERVAL: Duration = Duration::from_secs(60);
+// timeouts are fairly low as they will be multiplied with the amount
+// of sequential fetches without new posts by the scheduler.
+
+const DEFAULT_INTERVAL: Duration = Duration::from_secs(30);
 const MIN_INTERVAL: Duration = Duration::from_secs(10);
+const ERROR_INTERVAL: Duration = Duration::from_secs(180);
 
 #[derive(Debug)]
 pub enum Message {
@@ -12,7 +16,6 @@ pub enum Message {
         new_posts: usize,
         next_interval: Duration,
     },
-    Error { host: String },
     IntroduceHosts { hosts: Vec<String> },
 }
 
@@ -78,7 +81,11 @@ pub fn fetch_and_process(
             }
             Err(e) => {
                 log::error!("Failed fetching {}: {}", host, e);
-                message_tx.send(Message::Error { host }).unwrap();
+                message_tx.send(Message::Fetched {
+                    host,
+                    new_posts: 0,
+                    next_interval: ERROR_INTERVAL,
+                }).unwrap();
             }
         }
     });
diff --git a/nixos-module.nix b/nixos-module.nix
index 4385128..2ff89dc 100644
--- a/nixos-module.nix
+++ b/nixos-module.nix
@@ -6,7 +6,6 @@ let
   hunterDefaultSettings = {
     redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
     hosts = [ "mastodon.social" "fosstodon.org" "chaos.social" "dresden.network" ];
-    interval_after_error = 7200;
     max_workers = 16;
   };