This commit is contained in:
Astro 2024-03-27 02:11:35 +01:00
parent 3a14867a75
commit b06c0e48c5
15 changed files with 41 additions and 44 deletions

View File

@ -175,7 +175,7 @@ async fn run(
// print // print
for (period, keep, total, remove, add) in updates.clone() { for (period, keep, total, remove, add) in updates.clone() {
if add.len() > 0 || remove.len() > 0 { if !add.is_empty() || !remove.is_empty() {
tracing::info!("Trending in {:?} for {} of {}/{}: +{:?} -{:?}", language, period, keep, total, add, remove); tracing::info!("Trending in {:?} for {} of {}/{}: +{:?} -{:?}", language, period, keep, total, add, remove);
} }
} }

View File

@ -30,7 +30,7 @@ async fn follow_back(
let key_id = format!("{}#key", follow.actor); let key_id = format!("{}#key", follow.actor);
let result = activitypub::send::send( let result = activitypub::send::send(
client, &follow.inbox, client, &follow.inbox,
&key_id, &priv_key, &key_id, priv_key,
&action, &action,
).await; ).await;
match result { match result {

View File

@ -115,7 +115,7 @@ impl Post {
bot: actor.actor_type != "Person", bot: actor.actor_type != "Person",
}, },
tags: self.tag.into_iter().filter_map(|mut tag| { tags: self.tag.into_iter().filter_map(|mut tag| {
while tag.name.chars().next() == Some('#') { while tag.name.starts_with('#') {
tag.name.remove(0); tag.name.remove(0);
} }
if let Some(c) = tag.name.chars().next() { if let Some(c) = tag.name.chars().next() {

View File

@ -15,7 +15,6 @@ impl Leaf {
pub fn insert(&mut self, host: &str) { pub fn insert(&mut self, host: &str) {
match self { match self {
Leaf::Blocked => { Leaf::Blocked => {
return;
} }
&mut Leaf::Tree(ref mut tree) => { &mut Leaf::Tree(ref mut tree) => {
if let Some(pos) = host.rfind('.') { if let Some(pos) = host.rfind('.') {
@ -33,7 +32,7 @@ impl Leaf {
match self { match self {
Leaf::Blocked => Leaf::Blocked =>
true, true,
Leaf::Tree(ref tree) if host.len() > 0 => { Leaf::Tree(ref tree) if !host.is_empty() => {
let (label, new_host) = if let Some(pos) = host.rfind('.') { let (label, new_host) = if let Some(pos) = host.rfind('.') {
(&host[pos + 1..], &host[..pos]) (&host[pos + 1..], &host[..pos])
} else { } else {
@ -62,8 +61,8 @@ impl BlockList {
let mut root = Leaf::Tree(HashMap::new()); let mut root = Leaf::Tree(HashMap::new());
let mut file = BufReader::new(file); let mut file = BufReader::new(file);
let mut line = String::new(); let mut line = String::new();
while let Ok(_) = file.read_line(&mut line).await { while (file.read_line(&mut line).await).is_ok() {
if line == "" { if line.is_empty() {
break break
} }
@ -106,11 +105,11 @@ async fn test_blocklist() {
root.insert("bad.actor"); root.insert("bad.actor");
root.insert("evil.actor"); root.insert("evil.actor");
drop(root); drop(root);
assert_eq!(true, bl.is_blocked("bad.actor").await); assert!(bl.is_blocked("bad.actor").await);
assert_eq!(true, bl.is_blocked("evil.actor").await); assert!(bl.is_blocked("evil.actor").await);
assert_eq!(false, bl.is_blocked("good.actor").await); assert!(!bl.is_blocked("good.actor").await);
assert_eq!(false, bl.is_blocked("not-bad.actor").await); assert!(!bl.is_blocked("not-bad.actor").await);
assert_eq!(false, bl.is_blocked("actor").await); assert!(!bl.is_blocked("actor").await);
} }
#[cfg(test)] #[cfg(test)]
@ -121,6 +120,6 @@ async fn test_blocklist_subdomain() {
}; };
bl.tree.write().await bl.tree.write().await
.insert("bad.actor"); .insert("bad.actor");
assert_eq!(true, bl.is_blocked("bad.actor").await); assert!(bl.is_blocked("bad.actor").await);
assert_eq!(true, bl.is_blocked("very.bad.actor").await); assert!(bl.is_blocked("very.bad.actor").await);
} }

View File

@ -4,9 +4,7 @@ pub trait LoadConfig {
impl<T: Sized + for<'a> serde::Deserialize<'a>> LoadConfig for T { impl<T: Sized + for<'a> serde::Deserialize<'a>> LoadConfig for T {
fn load() -> Self { fn load() -> Self {
let path = std::env::args() let path = std::env::args().nth(1)
.skip(1)
.next()
.expect("Call with config.yaml"); .expect("Call with config.yaml");
crate::systemd::status(&format!("Loading config file {}", path)); crate::systemd::status(&format!("Loading config file {}", path));

View File

@ -19,5 +19,5 @@ pub const PERIODS: &[u64] = &[4, 24, 7 * 24];
pub const PERIOD_COMPARE_WINDOW: u64 = 3; pub const PERIOD_COMPARE_WINDOW: u64 = 3;
pub fn current_hour() -> u64 { pub fn current_hour() -> u64 {
chrono::offset::Utc::now().naive_utc().timestamp() as u64 / 3600 chrono::offset::Utc::now().timestamp() as u64 / 3600
} }

View File

@ -175,7 +175,7 @@ impl Store {
return; return;
} }
}; };
let hour = timestamp.naive_utc().timestamp() as u64 / 3600; let hour = timestamp.to_utc().timestamp() as u64 / 3600;
let until = current_hour(); let until = current_hour();
if hour > until { if hour > until {
tracing::warn!("future post from {}", timestamp); tracing::warn!("future post from {}", timestamp);
@ -209,7 +209,7 @@ impl Store {
).ignore(); ).ignore();
if let Some(user_id) = &user_id { if let Some(user_id) = &user_id {
// users by tag/hour // users by tag/hour
cmd.sadd(&user_key, &user_id).ignore() cmd.sadd(&user_key, user_id).ignore()
.expire(&user_key, TAG_EXPIRE as usize * 3600) .expire(&user_key, TAG_EXPIRE as usize * 3600)
.ignore(); .ignore();
} }
@ -380,7 +380,7 @@ impl Store {
let names = names.map(|name| { let names = names.map(|name| {
cmd.hgetall(tag_key(language, &name)); cmd.hgetall(tag_key(language, &name));
for hour in from..=until { for hour in from..=until {
cmd.scard(format!("u:{}:{}:{}", language.as_ref().map_or("", |l| &l), hour, name)); cmd.scard(format!("u:{}:{}:{}", language.as_ref().map_or("", |l| l), hour, name));
} }
name name
}).collect::<Vec<String>>(); }).collect::<Vec<String>>();

View File

@ -38,7 +38,7 @@ impl TrendTag {
pub fn score(&self, period: u64, until: u64) -> f64 { pub fn score(&self, period: u64, until: u64) -> f64 {
// ignore spam that comes from only 1 instance // ignore spam that comes from only 1 instance
if self.hosts().skip(1).next().is_none() { if self.hosts().nth(1).is_none() {
return -1.; return -1.;
} }
@ -48,7 +48,7 @@ impl TrendTag {
let mut before_hours = 0; let mut before_hours = 0;
let mut after_mentions = 0; let mut after_mentions = 0;
for (hour, mut mentions) in self.hour_users.iter().cloned() { for (hour, mentions) in self.hour_users.iter().cloned() {
if hour > from { if hour > from {
if mentions > 1 { if mentions > 1 {
after_mentions += mentions; after_mentions += mentions;

View File

@ -16,8 +16,8 @@ impl WordList {
let mut list = HashSet::new(); let mut list = HashSet::new();
let mut file = BufReader::new(file); let mut file = BufReader::new(file);
let mut line = String::new(); let mut line = String::new();
while let Ok(_) = file.read_line(&mut line).await { while (file.read_line(&mut line).await).is_ok() {
if line == "" { if line.is_empty() {
break break
} }

View File

@ -1,5 +1,5 @@
use std::{ use std::{
collections::{HashMap, HashSet}, net::SocketAddr, ops::Deref, pin::Pin collections::{HashMap, HashSet}, net::SocketAddr, ops::Deref
}; };
use askama::Template; use askama::Template;
use axum::{ use axum::{
@ -11,16 +11,16 @@ use axum::{
routing::get, routing::get,
Router, Router,
middleware::{Next, self}, middleware::{Next, self},
body::{Body, Bytes}, body::{Body},
}; };
use futures::{stream, Future, StreamExt}; use futures::{stream, StreamExt};
use futures::future::{join, join_all}; use futures::future::{join, join_all};
use cave::{ use cave::{
firehose::FirehoseFactory, firehose::FirehoseFactory,
store::{Store, TREND_POOL_SIZE}, PERIODS, store::{Store, TREND_POOL_SIZE}, PERIODS,
systemd, db::Database, systemd, db::Database,
}; };
use http_body::combinators::UnsyncBoxBody;
use metrics_exporter_prometheus::PrometheusHandle; use metrics_exporter_prometheus::PrometheusHandle;
use crate::{ use crate::{
html_template::HtmlTemplate, html_template::HtmlTemplate,
@ -186,7 +186,7 @@ async fn streaming_api(
.status(200) .status(200)
.header("content-type", "text/event-stream") .header("content-type", "text/event-stream")
.header("cache-control", "no-store") .header("cache-control", "no-store")
.body(body.into()) .body(body)
.expect("Response") .expect("Response")
} }

View File

@ -17,16 +17,16 @@ async fn collect_token(
code: String, code: String,
) -> Result<(), String> { ) -> Result<(), String> {
// try a few registered apps until one works // try a few registered apps until one works
for (client_id, client_secret) in db.get_apps(&host).await for (client_id, client_secret) in db.get_apps(host).await
.map_err(|e| format!("{}", e))? .map_err(|e| format!("{}", e))?
{ {
let app = oauth::Application { let app = oauth::Application {
client_id, client_id,
client_secret, client_secret,
}; };
match app.obtain_token(http_client, &host, code.clone()).await { match app.obtain_token(http_client, host, code.clone()).await {
Ok(token) => { Ok(token) => {
db.add_token(&host, &app.client_id, &token).await db.add_token(host, &app.client_id, &token).await
.expect("db.add_token"); .expect("db.add_token");
// success, done! // success, done!
return Ok(()); return Ok(());
@ -62,11 +62,11 @@ pub async fn get_token_collect(
} }
Err(e) => { Err(e) => {
tracing::error!("{}", e); tracing::error!("{}", e);
return ( (
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
[("content-type", "text/plain")], [("content-type", "text/plain")],
e e
).into_response(); ).into_response()
} }
} }
} }

View File

@ -111,10 +111,10 @@ impl TrendAnalyzer {
self.result.insert(ScoreKey { score, tag: name.clone(), }, tag.clone()); self.result.insert(ScoreKey { score, tag: name.clone(), }, tag.clone());
let mut least = self.result.keys().cloned().next().unwrap(); let mut least = self.result.keys().next().cloned().unwrap();
if self.result.len() > self.size { if self.result.len() > self.size {
self.result.remove(&least); self.result.remove(&least);
least = self.result.keys().cloned().next().unwrap().clone(); least = self.result.keys().next().cloned().unwrap().clone();
} }
self.score_threshold = Some(least.score); self.score_threshold = Some(least.score);

View File

@ -63,7 +63,7 @@ async fn run() {
.await.expect("get_hosts"); .await.expect("get_hosts");
pin_mut!(hosts); pin_mut!(hosts);
while let Some(host) = hosts.next().await { while let Some(host) = hosts.next().await {
if scheduler.introduce(InstanceHost::just_host(host.clone())).await == false { if ! scheduler.introduce(InstanceHost::just_host(host.clone())).await {
tracing::debug!("Remove host {}", host); tracing::debug!("Remove host {}", host);
store.remove_host(&host).await.expect("remove_host"); store.remove_host(&host).await.expect("remove_host");
} }

View File

@ -113,7 +113,7 @@ pub async fn run(
if let Ok(hosts) = webfinger_result { if let Ok(hosts) = webfinger_result {
for host in hosts { for host in hosts {
message_tx.send(Message::IntroduceHost(InstanceHost { message_tx.send(Message::IntroduceHost(InstanceHost {
host: host, host,
known_user: None, known_user: None,
})).unwrap(); })).unwrap();
} }
@ -182,7 +182,7 @@ async fn fetch_timeline(
metrics::increment_gauge!("hunter_requests", 1.0, "type" => "timeline"); metrics::increment_gauge!("hunter_requests", 1.0, "type" => "timeline");
let t1 = Instant::now(); let t1 = Instant::now();
let result = Feed::fetch(&client, &url).await; let result = Feed::fetch(client, &url).await;
let t2 = Instant::now(); let t2 = Instant::now();
metrics::decrement_gauge!("hunter_requests", 1.0, "type" => "timeline"); metrics::decrement_gauge!("hunter_requests", 1.0, "type" => "timeline");
metrics::histogram!("hunter_fetch_seconds", t2 - t1, "result" => if result.is_ok() { "ok" } else { "error" }); metrics::histogram!("hunter_fetch_seconds", t2 - t1, "result" => if result.is_ok() { "ok" } else { "error" });
@ -191,13 +191,13 @@ async fn fetch_timeline(
let mean_interval = feed.mean_post_interval(); let mean_interval = feed.mean_post_interval();
let (new_post_ratio, introduce_hosts) = process_posts(&mut store, posts_cache, block_list, &host, feed.posts.into_iter()).await; let (new_post_ratio, introduce_hosts) = process_posts(&mut store, posts_cache, block_list, host, feed.posts.into_iter()).await;
for introduce_host in introduce_hosts.into_iter() { for introduce_host in introduce_hosts.into_iter() {
message_tx.send(Message::IntroduceHost(introduce_host)).unwrap(); message_tx.send(Message::IntroduceHost(introduce_host)).unwrap();
} }
// successfully fetched, save for future run // successfully fetched, save for future run
store.save_host(&host).await.unwrap(); store.save_host(host).await.unwrap();
Ok((new_post_ratio, mean_interval)) Ok((new_post_ratio, mean_interval))
} }
@ -235,7 +235,7 @@ async fn process_posts(
scan_for_hosts(&mut introduce_hosts, &post); scan_for_hosts(&mut introduce_hosts, &post);
if let Some(reblog) = &post.reblog { if let Some(reblog) = &post.reblog {
scan_for_hosts(&mut introduce_hosts, &reblog); scan_for_hosts(&mut introduce_hosts, reblog);
} }
if let Some(account_host) = post.account.host() { if let Some(account_host) = post.account.host() {

View File

@ -57,7 +57,7 @@ fn html_to_text(html: &str) -> String {
fn language_colour(language: &str) -> Colour { fn language_colour(language: &str) -> Colour {
let x = language.bytes().fold(0, |x, b| x ^ b); let x = language.bytes().fold(0, |x, b| x ^ b);
let b = language.as_bytes(); let b = language.as_bytes();
let y = if b.len() >= 1 { let y = if !b.is_empty() {
(b[0] & 0x1F) << 2 (b[0] & 0x1F) << 2
} else { } else {
127 127