astro/caveman
astro
/
caveman
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

hunter: redesign hosts introduction in worker to add scanning reblog mentions

This commit is contained in:
Astro 2023-04-24 19:27:13 +02:00
parent 0bef428894
commit 33c910ff7a
1 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hunter/src/worker.rs
View File

@ -2,6 +2,7 @@ use std::collections::HashSet;
use std::future::Future;
use std::sync::Arc;
use std::time::{Duration, Instant};
use cave::feed::Post;
use cave::{
feed::{Feed, EncodablePost},
store::Store,
@ -192,25 +193,28 @@ async fn process_posts(
if ! posts_cache.insert(post.uri.clone()) {
let t1 = Instant::now();
// introduce instances from reblog authors
if let Some(reblog_account_host) = post.reblog.as_ref().and_then(|reblog| reblog.account.host()) {
introduce_hosts.insert(reblog_account_host);
}
// introduce instances from mentions
for mention in &post.mentions {
if let Some(user_host) = mention.user_host() {
introduce_hosts.insert(user_host);
fn scan_for_hosts(introduce_hosts: &mut HashSet<String>, post: &Post) {
// introduce instances from accounts
if let Some(account_host) = post.account.host() {
introduce_hosts.insert(account_host);
}
// introduce instances from mentions
for mention in &post.mentions {
if let Some(user_host) = mention.user_host() {
introduce_hosts.insert(user_host);
}
}
}
scan_for_hosts(&mut introduce_hosts, &post);
if let Some(reblog) = &post.reblog {
scan_for_hosts(&mut introduce_hosts, &reblog);
}
// check if it's an actual post
if let Some(account_host) = post.account.host() {
if let Some(_account_host) = post.account.host() {
// send away to redis
if store.save_post(post).await == Ok(true) {
new_posts += 1;
}
// introduce instances from accounts
introduce_hosts.insert(account_host);
} else {
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
}