astro/caveman
astro/caveman
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

hunter/worker: use url not uri as key (same as in activitypub)

Browse Source
This commit is contained in:
Astro 2023-11-10 23:33:01 +01:00
parent 8d2fcbcb9e
commit 03968b9cbb
4 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cave/src/activitypub/mod.rs
View File

@ -106,7 +106,7 @@ impl Post {
.map(|s| s.to_string()); .map(|s| s.to_string());
super::feed::Post { super::feed::Post {
created_at: self.published, created_at: self.published,
uri: self.url.unwrap_or(self.id), url: self.url.unwrap_or(self.id),
content: self.content, content: self.content,
account: super::feed::Account { account: super::feed::Account {
username: actor.preferred_username, username: actor.preferred_username,

10
cave/src/feed.rs
View File

@ -64,7 +64,7 @@ pub struct MediaAttachment {
#[derive(Debug, serde::Serialize, serde::Deserialize)] #[derive(Debug, serde::Serialize, serde::Deserialize)]
pub struct Post { pub struct Post {
pub created_at: String, pub created_at: String,
pub uri: String, pub url: String,
#[serde(default = "String::new")] #[serde(default = "String::new")]
pub content: String, pub content: String,
pub account: Account, pub account: Account,
@ -82,17 +82,17 @@ pub struct Post {
} }
impl Post { impl Post {
pub fn uri_host(&self) -> Option<String> { pub fn url_host(&self) -> Option<String> {
reqwest::Url::parse(&self.uri) reqwest::Url::parse(&self.url)
.ok() .ok()
.and_then(|uri| uri.domain() .and_then(|url| url.domain()
.map(|host| host.to_owned()) .map(|host| host.to_owned())
) )
} }
pub fn user_id(&self) -> Option<String> { pub fn user_id(&self) -> Option<String> {
let username = self.account.username.to_lowercase(); let username = self.account.username.to_lowercase();
let host = self.uri_host()?; let host = self.url_host()?;
Some(format!("{}@{}", username, host)) Some(format!("{}@{}", username, host))
} }

8
cave/src/store.rs
View File

@ -118,7 +118,7 @@ impl Store {
} }
pub async fn save_post(&mut self, mut post: EncodablePost) -> Result<bool, RedisError> { pub async fn save_post(&mut self, mut post: EncodablePost) -> Result<bool, RedisError> {
let post_key = format!("p:{}", post.uri); let post_key = format!("p:{}", post.url);
let check = redis::pipe() let check = redis::pipe()
.getset(&post_key, "1") .getset(&post_key, "1")
.expire(post_key, POST_EXPIRE) .expire(post_key, POST_EXPIRE)
@ -132,7 +132,7 @@ impl Store {
tracing::info!("New post ({}{} tags): {}", tracing::info!("New post ({}{} tags): {}",
if post.account.bot { "bot, " } else { "" }, if post.account.bot { "bot, " } else { "" },
post.tags.len(), post.uri); post.tags.len(), post.url);
match post.encode() { match post.encode() {
Ok(encoded_post) => { Ok(encoded_post) => {
@ -160,10 +160,10 @@ impl Store {
return; return;
} }
let host = match post.uri_host() { let host = match post.url_host() {
Some(host) => host, Some(host) => host,
None => { None => {
tracing::warn!("no uri_host"); tracing::warn!("no url_host");
return; return;
}, },
}; };

2
hunter/src/worker.rs
View File

@ -230,7 +230,7 @@ async fn process_posts(
posts_len += 1; posts_len += 1;
// potentially save a round-trip to redis with an in-process cache // potentially save a round-trip to redis with an in-process cache
if ! posts_cache.insert(post.uri.clone()) { if ! posts_cache.insert(post.url.clone()) {
let t1 = Instant::now(); let t1 = Instant::now();
scan_for_hosts(&mut introduce_hosts, &post); scan_for_hosts(&mut introduce_hosts, &post);