110 lines
2.7 KiB
Rust
110 lines
2.7 KiB
Rust
use crate::feed::{Post, Tag};
|
|
|
|
const POST_EXPIRE: usize = 86400;
|
|
|
|
pub async fn save_post(man: &mut redis::aio::ConnectionManager, host: &str, post: Post) -> bool {
|
|
let post_key = format!("p:{}", post.uri);
|
|
let check = redis::pipe()
|
|
.getset(&post_key, "1")
|
|
.expire(post_key, POST_EXPIRE)
|
|
.ignore()
|
|
.query_async::<_, redis::Value>(man)
|
|
.await
|
|
.unwrap();
|
|
if check != redis::Value::Bulk(vec![redis::Value::Nil]) {
|
|
// post is not new
|
|
return false;
|
|
}
|
|
log::info!("New post ({}): {}", post.tags.len(), post.uri);
|
|
|
|
save_post_tags(man, post).await;
|
|
|
|
// post was new
|
|
true
|
|
}
|
|
|
|
async fn save_post_tags(man: &mut redis::aio::ConnectionManager, post: Post) {
|
|
if post.account.bot || post.tags.is_empty() {
|
|
// irrelevant
|
|
return;
|
|
}
|
|
|
|
let host = match post.uri_host() {
|
|
Some(host) => host,
|
|
None => return,
|
|
};
|
|
|
|
let timestamp = match post.timestamp() {
|
|
Some(timestamp) => timestamp,
|
|
None => return,
|
|
};
|
|
let hour = timestamp.naive_utc().timestamp() / 3600;
|
|
|
|
let mut cmd = redis::pipe();
|
|
let mut store_tags = |tag: &Tag, tag_key| {
|
|
// by hour
|
|
cmd.hincr(
|
|
&tag_key,
|
|
format!("t:{}", hour),
|
|
1
|
|
).ignore();
|
|
// by spelling
|
|
cmd.hincr(
|
|
&tag_key,
|
|
format!("s:{}", tag.name),
|
|
1
|
|
).ignore();
|
|
// by instance
|
|
cmd.hincr(
|
|
tag_key,
|
|
format!("h:{}", host),
|
|
1
|
|
).ignore();
|
|
};
|
|
for tag in post.tags {
|
|
// global
|
|
store_tags(&tag, format!("g:{}", tag.name.to_lowercase()));
|
|
// by language
|
|
if let Some(language) = &post.language {
|
|
store_tags(&tag, format!("l:{}:{}", language, tag.name.to_lowercase()));
|
|
}
|
|
}
|
|
|
|
match cmd.query_async(man).await {
|
|
Ok(()) => {}
|
|
Err(e) => {
|
|
log::error!("redis error: {:?}", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn save_host(man: &mut redis::aio::ConnectionManager, host: &str) {
|
|
redis::Cmd::set(format!("h:{}", host), "1")
|
|
.query_async::<_, ()>(man)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
pub async fn get_hosts(
|
|
man: &mut redis::aio::ConnectionManager,
|
|
) -> Vec<String> {
|
|
let mut results = vec![];
|
|
scan(man, "h:*", |key| results.push(key[2..].to_string())).await;
|
|
results
|
|
}
|
|
|
|
async fn scan<F: FnMut(String)>(
|
|
man: &mut redis::aio::ConnectionManager,
|
|
pattern: &str,
|
|
mut f: F,
|
|
) {
|
|
let mut cmd = redis::cmd("SCAN");
|
|
cmd.cursor_arg(0).arg("MATCH").arg(pattern);
|
|
let mut iter = cmd.iter_async::<String>(man)
|
|
.await
|
|
.unwrap();
|
|
while let Some(key) = iter.next_item().await {
|
|
f(key);
|
|
}
|
|
}
|