hunter: redesign hosts introduction in worker to add scanning reblog mentions
This commit is contained in:
parent
0bef428894
commit
33c910ff7a
|
@ -2,6 +2,7 @@ use std::collections::HashSet;
|
|||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use cave::feed::Post;
|
||||
use cave::{
|
||||
feed::{Feed, EncodablePost},
|
||||
store::Store,
|
||||
|
@ -192,9 +193,10 @@ async fn process_posts(
|
|||
if ! posts_cache.insert(post.uri.clone()) {
|
||||
let t1 = Instant::now();
|
||||
|
||||
// introduce instances from reblog authors
|
||||
if let Some(reblog_account_host) = post.reblog.as_ref().and_then(|reblog| reblog.account.host()) {
|
||||
introduce_hosts.insert(reblog_account_host);
|
||||
fn scan_for_hosts(introduce_hosts: &mut HashSet<String>, post: &Post) {
|
||||
// introduce instances from accounts
|
||||
if let Some(account_host) = post.account.host() {
|
||||
introduce_hosts.insert(account_host);
|
||||
}
|
||||
// introduce instances from mentions
|
||||
for mention in &post.mentions {
|
||||
|
@ -202,15 +204,17 @@ async fn process_posts(
|
|||
introduce_hosts.insert(user_host);
|
||||
}
|
||||
}
|
||||
}
|
||||
scan_for_hosts(&mut introduce_hosts, &post);
|
||||
if let Some(reblog) = &post.reblog {
|
||||
scan_for_hosts(&mut introduce_hosts, &reblog);
|
||||
}
|
||||
|
||||
// check if it's an actual post
|
||||
if let Some(account_host) = post.account.host() {
|
||||
if let Some(_account_host) = post.account.host() {
|
||||
// send away to redis
|
||||
if store.save_post(post).await == Ok(true) {
|
||||
new_posts += 1;
|
||||
}
|
||||
// introduce instances from accounts
|
||||
introduce_hosts.insert(account_host);
|
||||
} else {
|
||||
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue