caveman/src/worker.rs

use std::collections::HashSet;
use crate::feed;

#[derive(Debug)]
pub enum Message {
    Fetched { host: String },
    Error { host: String },
    IntroduceHosts { hosts: Vec<String> },
    Posts { posts: Vec<feed::Post> },
}

pub fn fetch_and_process(
    tx: tokio::sync::mpsc::UnboundedSender<Message>,
    client: reqwest::Client,
    host: String,
) {
    let url = format!("https://{}/api/v1/timelines/public", host);
    tokio::spawn(async move {
        match feed::Feed::fetch(&client, &url).await {
            Ok(feed) => {
                tx.send(Message::Fetched { host: host.clone() }).unwrap();

                // Analyze
                let mut posts = Vec::with_capacity(feed.posts.len());
                let mut hosts = HashSet::new();
                for post in feed.posts.into_iter() {
                    if let Some(author_host) = post.account.host() {
                        if author_host == host && post.url_host().as_ref() == Some(&host) {
                            posts.push(post);
                        }
                        hosts.insert(author_host);
                    }
                }
                let hosts = hosts.into_iter().collect();
                tx.send(Message::IntroduceHosts { hosts }).unwrap();
                tx.send(Message::Posts { posts }).unwrap();
            }
            Err(e) => {
                println!("Failed fetching {}: {}", host, e);
                tx.send(Message::Error { host }).unwrap();
            }
        }
    });
}