astro/caveman
astro
/
caveman
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

hunter: redesign hosts introduction in worker to add scanning reblog mentions

This commit is contained in:
Astro 2023-04-24 19:27:13 +02:00
parent 0bef428894
commit 33c910ff7a
1 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hunter/src/worker.rs
View File

@ -2,6 +2,7 @@ use std::collections::HashSet;
use std::future::Future; use std::future::Future;
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use cave::feed::Post;
use cave::{ use cave::{
feed::{Feed, EncodablePost}, feed::{Feed, EncodablePost},
store::Store, store::Store,
@ -192,25 +193,28 @@ async fn process_posts(
if ! posts_cache.insert(post.uri.clone()) { if ! posts_cache.insert(post.uri.clone()) {
let t1 = Instant::now(); let t1 = Instant::now();
// introduce instances from reblog authors fn scan_for_hosts(introduce_hosts: &mut HashSet<String>, post: &Post) {
if let Some(reblog_account_host) = post.reblog.as_ref().and_then(|reblog| reblog.account.host()) { // introduce instances from accounts
introduce_hosts.insert(reblog_account_host); if let Some(account_host) = post.account.host() {
} introduce_hosts.insert(account_host);
// introduce instances from mentions }
for mention in &post.mentions { // introduce instances from mentions
if let Some(user_host) = mention.user_host() { for mention in &post.mentions {
introduce_hosts.insert(user_host); if let Some(user_host) = mention.user_host() {
introduce_hosts.insert(user_host);
}
} }
} }
scan_for_hosts(&mut introduce_hosts, &post);
if let Some(reblog) = &post.reblog {
scan_for_hosts(&mut introduce_hosts, &reblog);
}
// check if it's an actual post if let Some(_account_host) = post.account.host() {
if let Some(account_host) = post.account.host() {
// send away to redis // send away to redis
if store.save_post(post).await == Ok(true) { if store.save_post(post).await == Ok(true) {
new_posts += 1; new_posts += 1;
} }
// introduce instances from accounts
introduce_hosts.insert(account_host);
} else { } else {
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host); tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
} }