89 lines
2.3 KiB
Rust
89 lines
2.3 KiB
Rust
use std::{
|
|
sync::Arc,
|
|
};
|
|
use futures::StreamExt;
|
|
use cave::{
|
|
config::LoadConfig,
|
|
feed::Post,
|
|
firehose::FirehoseFactory,
|
|
word_list::WordList,
|
|
};
|
|
use trend_setter::UpdateSet;
|
|
|
|
mod config;
|
|
mod trend_setter;
|
|
|
|
#[global_allocator]
|
|
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
|
|
|
async fn is_profane(profanity: &WordList, post: &Post) -> bool {
|
|
if post.sensitive == Some(true) {
|
|
return true;
|
|
}
|
|
|
|
let tags_set = post.tags_set();
|
|
let tagged_profanity =
|
|
futures::stream::iter(
|
|
tags_set.iter()
|
|
)
|
|
.any(|(tag, _spellings)| profanity.contains(tag));
|
|
|
|
tagged_profanity.await
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() {
|
|
cave::init::exit_on_panic();
|
|
cave::init::init_logger(5555);
|
|
|
|
let config = config::Config::load();
|
|
let profanity = WordList::new(&config.profanity).await;
|
|
|
|
let store = cave::store::Store::new(
|
|
16, config.redis.clone(), config.redis_password_file.clone()
|
|
).await;
|
|
|
|
cave::systemd::status("Starting trend_setter");
|
|
let trend_setter_tx = trend_setter::start(store.clone());
|
|
|
|
let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file);
|
|
let firehose = firehose_factory.produce()
|
|
.await
|
|
.expect("firehose");
|
|
cave::systemd::ready();
|
|
|
|
firehose.for_each(move |(event_type, data)| {
|
|
if event_type != b"update" {
|
|
// Only analyze new posts, no updates
|
|
return futures::future::ready(());
|
|
}
|
|
|
|
let trend_setter_tx = trend_setter_tx.clone();
|
|
let mut store = store.clone();
|
|
let profanity = profanity.clone();
|
|
tokio::spawn(async move {
|
|
let post = match serde_json::from_slice(&data) {
|
|
Ok(post) =>
|
|
post,
|
|
Err(e) => {
|
|
tracing::error!("Cannot parse JSON: {:?}", e);
|
|
return;
|
|
},
|
|
};
|
|
let post = Arc::new(post);
|
|
store.save_post_tags(&post, is_profane(&profanity, &post).await).await;
|
|
|
|
let update_set = UpdateSet::from(&*post);
|
|
if ! update_set.is_empty() {
|
|
trend_setter_tx.send(update_set).await.unwrap();
|
|
}
|
|
|
|
cave::systemd::watchdog();
|
|
});
|
|
|
|
futures::future::ready(())
|
|
}).await;
|
|
|
|
panic!("End")
|
|
}
|