delint
This commit is contained in:
parent
3a14867a75
commit
b06c0e48c5
|
@ -175,7 +175,7 @@ async fn run(
|
||||||
|
|
||||||
// print
|
// print
|
||||||
for (period, keep, total, remove, add) in updates.clone() {
|
for (period, keep, total, remove, add) in updates.clone() {
|
||||||
if add.len() > 0 || remove.len() > 0 {
|
if !add.is_empty() || !remove.is_empty() {
|
||||||
tracing::info!("Trending in {:?} for {} of {}/{}: +{:?} -{:?}", language, period, keep, total, add, remove);
|
tracing::info!("Trending in {:?} for {} of {}/{}: +{:?} -{:?}", language, period, keep, total, add, remove);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ async fn follow_back(
|
||||||
let key_id = format!("{}#key", follow.actor);
|
let key_id = format!("{}#key", follow.actor);
|
||||||
let result = activitypub::send::send(
|
let result = activitypub::send::send(
|
||||||
client, &follow.inbox,
|
client, &follow.inbox,
|
||||||
&key_id, &priv_key,
|
&key_id, priv_key,
|
||||||
&action,
|
&action,
|
||||||
).await;
|
).await;
|
||||||
match result {
|
match result {
|
||||||
|
|
|
@ -115,7 +115,7 @@ impl Post {
|
||||||
bot: actor.actor_type != "Person",
|
bot: actor.actor_type != "Person",
|
||||||
},
|
},
|
||||||
tags: self.tag.into_iter().filter_map(|mut tag| {
|
tags: self.tag.into_iter().filter_map(|mut tag| {
|
||||||
while tag.name.chars().next() == Some('#') {
|
while tag.name.starts_with('#') {
|
||||||
tag.name.remove(0);
|
tag.name.remove(0);
|
||||||
}
|
}
|
||||||
if let Some(c) = tag.name.chars().next() {
|
if let Some(c) = tag.name.chars().next() {
|
||||||
|
|
|
@ -15,7 +15,6 @@ impl Leaf {
|
||||||
pub fn insert(&mut self, host: &str) {
|
pub fn insert(&mut self, host: &str) {
|
||||||
match self {
|
match self {
|
||||||
Leaf::Blocked => {
|
Leaf::Blocked => {
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
&mut Leaf::Tree(ref mut tree) => {
|
&mut Leaf::Tree(ref mut tree) => {
|
||||||
if let Some(pos) = host.rfind('.') {
|
if let Some(pos) = host.rfind('.') {
|
||||||
|
@ -33,7 +32,7 @@ impl Leaf {
|
||||||
match self {
|
match self {
|
||||||
Leaf::Blocked =>
|
Leaf::Blocked =>
|
||||||
true,
|
true,
|
||||||
Leaf::Tree(ref tree) if host.len() > 0 => {
|
Leaf::Tree(ref tree) if !host.is_empty() => {
|
||||||
let (label, new_host) = if let Some(pos) = host.rfind('.') {
|
let (label, new_host) = if let Some(pos) = host.rfind('.') {
|
||||||
(&host[pos + 1..], &host[..pos])
|
(&host[pos + 1..], &host[..pos])
|
||||||
} else {
|
} else {
|
||||||
|
@ -62,8 +61,8 @@ impl BlockList {
|
||||||
let mut root = Leaf::Tree(HashMap::new());
|
let mut root = Leaf::Tree(HashMap::new());
|
||||||
let mut file = BufReader::new(file);
|
let mut file = BufReader::new(file);
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
while let Ok(_) = file.read_line(&mut line).await {
|
while (file.read_line(&mut line).await).is_ok() {
|
||||||
if line == "" {
|
if line.is_empty() {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,11 +105,11 @@ async fn test_blocklist() {
|
||||||
root.insert("bad.actor");
|
root.insert("bad.actor");
|
||||||
root.insert("evil.actor");
|
root.insert("evil.actor");
|
||||||
drop(root);
|
drop(root);
|
||||||
assert_eq!(true, bl.is_blocked("bad.actor").await);
|
assert!(bl.is_blocked("bad.actor").await);
|
||||||
assert_eq!(true, bl.is_blocked("evil.actor").await);
|
assert!(bl.is_blocked("evil.actor").await);
|
||||||
assert_eq!(false, bl.is_blocked("good.actor").await);
|
assert!(!bl.is_blocked("good.actor").await);
|
||||||
assert_eq!(false, bl.is_blocked("not-bad.actor").await);
|
assert!(!bl.is_blocked("not-bad.actor").await);
|
||||||
assert_eq!(false, bl.is_blocked("actor").await);
|
assert!(!bl.is_blocked("actor").await);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -121,6 +120,6 @@ async fn test_blocklist_subdomain() {
|
||||||
};
|
};
|
||||||
bl.tree.write().await
|
bl.tree.write().await
|
||||||
.insert("bad.actor");
|
.insert("bad.actor");
|
||||||
assert_eq!(true, bl.is_blocked("bad.actor").await);
|
assert!(bl.is_blocked("bad.actor").await);
|
||||||
assert_eq!(true, bl.is_blocked("very.bad.actor").await);
|
assert!(bl.is_blocked("very.bad.actor").await);
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,7 @@ pub trait LoadConfig {
|
||||||
|
|
||||||
impl<T: Sized + for<'a> serde::Deserialize<'a>> LoadConfig for T {
|
impl<T: Sized + for<'a> serde::Deserialize<'a>> LoadConfig for T {
|
||||||
fn load() -> Self {
|
fn load() -> Self {
|
||||||
let path = std::env::args()
|
let path = std::env::args().nth(1)
|
||||||
.skip(1)
|
|
||||||
.next()
|
|
||||||
.expect("Call with config.yaml");
|
.expect("Call with config.yaml");
|
||||||
crate::systemd::status(&format!("Loading config file {}", path));
|
crate::systemd::status(&format!("Loading config file {}", path));
|
||||||
|
|
||||||
|
|
|
@ -19,5 +19,5 @@ pub const PERIODS: &[u64] = &[4, 24, 7 * 24];
|
||||||
pub const PERIOD_COMPARE_WINDOW: u64 = 3;
|
pub const PERIOD_COMPARE_WINDOW: u64 = 3;
|
||||||
|
|
||||||
pub fn current_hour() -> u64 {
|
pub fn current_hour() -> u64 {
|
||||||
chrono::offset::Utc::now().naive_utc().timestamp() as u64 / 3600
|
chrono::offset::Utc::now().timestamp() as u64 / 3600
|
||||||
}
|
}
|
||||||
|
|
|
@ -175,7 +175,7 @@ impl Store {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let hour = timestamp.naive_utc().timestamp() as u64 / 3600;
|
let hour = timestamp.to_utc().timestamp() as u64 / 3600;
|
||||||
let until = current_hour();
|
let until = current_hour();
|
||||||
if hour > until {
|
if hour > until {
|
||||||
tracing::warn!("future post from {}", timestamp);
|
tracing::warn!("future post from {}", timestamp);
|
||||||
|
@ -209,7 +209,7 @@ impl Store {
|
||||||
).ignore();
|
).ignore();
|
||||||
if let Some(user_id) = &user_id {
|
if let Some(user_id) = &user_id {
|
||||||
// users by tag/hour
|
// users by tag/hour
|
||||||
cmd.sadd(&user_key, &user_id).ignore()
|
cmd.sadd(&user_key, user_id).ignore()
|
||||||
.expire(&user_key, TAG_EXPIRE as usize * 3600)
|
.expire(&user_key, TAG_EXPIRE as usize * 3600)
|
||||||
.ignore();
|
.ignore();
|
||||||
}
|
}
|
||||||
|
@ -380,7 +380,7 @@ impl Store {
|
||||||
let names = names.map(|name| {
|
let names = names.map(|name| {
|
||||||
cmd.hgetall(tag_key(language, &name));
|
cmd.hgetall(tag_key(language, &name));
|
||||||
for hour in from..=until {
|
for hour in from..=until {
|
||||||
cmd.scard(format!("u:{}:{}:{}", language.as_ref().map_or("", |l| &l), hour, name));
|
cmd.scard(format!("u:{}:{}:{}", language.as_ref().map_or("", |l| l), hour, name));
|
||||||
}
|
}
|
||||||
name
|
name
|
||||||
}).collect::<Vec<String>>();
|
}).collect::<Vec<String>>();
|
||||||
|
|
|
@ -38,7 +38,7 @@ impl TrendTag {
|
||||||
|
|
||||||
pub fn score(&self, period: u64, until: u64) -> f64 {
|
pub fn score(&self, period: u64, until: u64) -> f64 {
|
||||||
// ignore spam that comes from only 1 instance
|
// ignore spam that comes from only 1 instance
|
||||||
if self.hosts().skip(1).next().is_none() {
|
if self.hosts().nth(1).is_none() {
|
||||||
return -1.;
|
return -1.;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,7 +48,7 @@ impl TrendTag {
|
||||||
let mut before_hours = 0;
|
let mut before_hours = 0;
|
||||||
let mut after_mentions = 0;
|
let mut after_mentions = 0;
|
||||||
|
|
||||||
for (hour, mut mentions) in self.hour_users.iter().cloned() {
|
for (hour, mentions) in self.hour_users.iter().cloned() {
|
||||||
if hour > from {
|
if hour > from {
|
||||||
if mentions > 1 {
|
if mentions > 1 {
|
||||||
after_mentions += mentions;
|
after_mentions += mentions;
|
||||||
|
|
|
@ -16,8 +16,8 @@ impl WordList {
|
||||||
let mut list = HashSet::new();
|
let mut list = HashSet::new();
|
||||||
let mut file = BufReader::new(file);
|
let mut file = BufReader::new(file);
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
while let Ok(_) = file.read_line(&mut line).await {
|
while (file.read_line(&mut line).await).is_ok() {
|
||||||
if line == "" {
|
if line.is_empty() {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet}, net::SocketAddr, ops::Deref, pin::Pin
|
collections::{HashMap, HashSet}, net::SocketAddr, ops::Deref
|
||||||
};
|
};
|
||||||
use askama::Template;
|
use askama::Template;
|
||||||
use axum::{
|
use axum::{
|
||||||
|
@ -11,16 +11,16 @@ use axum::{
|
||||||
routing::get,
|
routing::get,
|
||||||
Router,
|
Router,
|
||||||
middleware::{Next, self},
|
middleware::{Next, self},
|
||||||
body::{Body, Bytes},
|
body::{Body},
|
||||||
};
|
};
|
||||||
use futures::{stream, Future, StreamExt};
|
use futures::{stream, StreamExt};
|
||||||
use futures::future::{join, join_all};
|
use futures::future::{join, join_all};
|
||||||
use cave::{
|
use cave::{
|
||||||
firehose::FirehoseFactory,
|
firehose::FirehoseFactory,
|
||||||
store::{Store, TREND_POOL_SIZE}, PERIODS,
|
store::{Store, TREND_POOL_SIZE}, PERIODS,
|
||||||
systemd, db::Database,
|
systemd, db::Database,
|
||||||
};
|
};
|
||||||
use http_body::combinators::UnsyncBoxBody;
|
|
||||||
use metrics_exporter_prometheus::PrometheusHandle;
|
use metrics_exporter_prometheus::PrometheusHandle;
|
||||||
use crate::{
|
use crate::{
|
||||||
html_template::HtmlTemplate,
|
html_template::HtmlTemplate,
|
||||||
|
@ -186,7 +186,7 @@ async fn streaming_api(
|
||||||
.status(200)
|
.status(200)
|
||||||
.header("content-type", "text/event-stream")
|
.header("content-type", "text/event-stream")
|
||||||
.header("cache-control", "no-store")
|
.header("cache-control", "no-store")
|
||||||
.body(body.into())
|
.body(body)
|
||||||
.expect("Response")
|
.expect("Response")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,16 +17,16 @@ async fn collect_token(
|
||||||
code: String,
|
code: String,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
// try a few registered apps until one works
|
// try a few registered apps until one works
|
||||||
for (client_id, client_secret) in db.get_apps(&host).await
|
for (client_id, client_secret) in db.get_apps(host).await
|
||||||
.map_err(|e| format!("{}", e))?
|
.map_err(|e| format!("{}", e))?
|
||||||
{
|
{
|
||||||
let app = oauth::Application {
|
let app = oauth::Application {
|
||||||
client_id,
|
client_id,
|
||||||
client_secret,
|
client_secret,
|
||||||
};
|
};
|
||||||
match app.obtain_token(http_client, &host, code.clone()).await {
|
match app.obtain_token(http_client, host, code.clone()).await {
|
||||||
Ok(token) => {
|
Ok(token) => {
|
||||||
db.add_token(&host, &app.client_id, &token).await
|
db.add_token(host, &app.client_id, &token).await
|
||||||
.expect("db.add_token");
|
.expect("db.add_token");
|
||||||
// success, done!
|
// success, done!
|
||||||
return Ok(());
|
return Ok(());
|
||||||
|
@ -62,11 +62,11 @@ pub async fn get_token_collect(
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("{}", e);
|
tracing::error!("{}", e);
|
||||||
return (
|
(
|
||||||
StatusCode::INTERNAL_SERVER_ERROR,
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
[("content-type", "text/plain")],
|
[("content-type", "text/plain")],
|
||||||
e
|
e
|
||||||
).into_response();
|
).into_response()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -111,10 +111,10 @@ impl TrendAnalyzer {
|
||||||
|
|
||||||
self.result.insert(ScoreKey { score, tag: name.clone(), }, tag.clone());
|
self.result.insert(ScoreKey { score, tag: name.clone(), }, tag.clone());
|
||||||
|
|
||||||
let mut least = self.result.keys().cloned().next().unwrap();
|
let mut least = self.result.keys().next().cloned().unwrap();
|
||||||
if self.result.len() > self.size {
|
if self.result.len() > self.size {
|
||||||
self.result.remove(&least);
|
self.result.remove(&least);
|
||||||
least = self.result.keys().cloned().next().unwrap().clone();
|
least = self.result.keys().next().cloned().unwrap().clone();
|
||||||
}
|
}
|
||||||
|
|
||||||
self.score_threshold = Some(least.score);
|
self.score_threshold = Some(least.score);
|
||||||
|
|
|
@ -63,7 +63,7 @@ async fn run() {
|
||||||
.await.expect("get_hosts");
|
.await.expect("get_hosts");
|
||||||
pin_mut!(hosts);
|
pin_mut!(hosts);
|
||||||
while let Some(host) = hosts.next().await {
|
while let Some(host) = hosts.next().await {
|
||||||
if scheduler.introduce(InstanceHost::just_host(host.clone())).await == false {
|
if ! scheduler.introduce(InstanceHost::just_host(host.clone())).await {
|
||||||
tracing::debug!("Remove host {}", host);
|
tracing::debug!("Remove host {}", host);
|
||||||
store.remove_host(&host).await.expect("remove_host");
|
store.remove_host(&host).await.expect("remove_host");
|
||||||
}
|
}
|
||||||
|
|
|
@ -113,7 +113,7 @@ pub async fn run(
|
||||||
if let Ok(hosts) = webfinger_result {
|
if let Ok(hosts) = webfinger_result {
|
||||||
for host in hosts {
|
for host in hosts {
|
||||||
message_tx.send(Message::IntroduceHost(InstanceHost {
|
message_tx.send(Message::IntroduceHost(InstanceHost {
|
||||||
host: host,
|
host,
|
||||||
known_user: None,
|
known_user: None,
|
||||||
})).unwrap();
|
})).unwrap();
|
||||||
}
|
}
|
||||||
|
@ -182,7 +182,7 @@ async fn fetch_timeline(
|
||||||
|
|
||||||
metrics::increment_gauge!("hunter_requests", 1.0, "type" => "timeline");
|
metrics::increment_gauge!("hunter_requests", 1.0, "type" => "timeline");
|
||||||
let t1 = Instant::now();
|
let t1 = Instant::now();
|
||||||
let result = Feed::fetch(&client, &url).await;
|
let result = Feed::fetch(client, &url).await;
|
||||||
let t2 = Instant::now();
|
let t2 = Instant::now();
|
||||||
metrics::decrement_gauge!("hunter_requests", 1.0, "type" => "timeline");
|
metrics::decrement_gauge!("hunter_requests", 1.0, "type" => "timeline");
|
||||||
metrics::histogram!("hunter_fetch_seconds", t2 - t1, "result" => if result.is_ok() { "ok" } else { "error" });
|
metrics::histogram!("hunter_fetch_seconds", t2 - t1, "result" => if result.is_ok() { "ok" } else { "error" });
|
||||||
|
@ -191,13 +191,13 @@ async fn fetch_timeline(
|
||||||
|
|
||||||
let mean_interval = feed.mean_post_interval();
|
let mean_interval = feed.mean_post_interval();
|
||||||
|
|
||||||
let (new_post_ratio, introduce_hosts) = process_posts(&mut store, posts_cache, block_list, &host, feed.posts.into_iter()).await;
|
let (new_post_ratio, introduce_hosts) = process_posts(&mut store, posts_cache, block_list, host, feed.posts.into_iter()).await;
|
||||||
for introduce_host in introduce_hosts.into_iter() {
|
for introduce_host in introduce_hosts.into_iter() {
|
||||||
message_tx.send(Message::IntroduceHost(introduce_host)).unwrap();
|
message_tx.send(Message::IntroduceHost(introduce_host)).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
// successfully fetched, save for future run
|
// successfully fetched, save for future run
|
||||||
store.save_host(&host).await.unwrap();
|
store.save_host(host).await.unwrap();
|
||||||
|
|
||||||
Ok((new_post_ratio, mean_interval))
|
Ok((new_post_ratio, mean_interval))
|
||||||
}
|
}
|
||||||
|
@ -235,7 +235,7 @@ async fn process_posts(
|
||||||
|
|
||||||
scan_for_hosts(&mut introduce_hosts, &post);
|
scan_for_hosts(&mut introduce_hosts, &post);
|
||||||
if let Some(reblog) = &post.reblog {
|
if let Some(reblog) = &post.reblog {
|
||||||
scan_for_hosts(&mut introduce_hosts, &reblog);
|
scan_for_hosts(&mut introduce_hosts, reblog);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(account_host) = post.account.host() {
|
if let Some(account_host) = post.account.host() {
|
||||||
|
|
|
@ -57,7 +57,7 @@ fn html_to_text(html: &str) -> String {
|
||||||
fn language_colour(language: &str) -> Colour {
|
fn language_colour(language: &str) -> Colour {
|
||||||
let x = language.bytes().fold(0, |x, b| x ^ b);
|
let x = language.bytes().fold(0, |x, b| x ^ b);
|
||||||
let b = language.as_bytes();
|
let b = language.as_bytes();
|
||||||
let y = if b.len() >= 1 {
|
let y = if !b.is_empty() {
|
||||||
(b[0] & 0x1F) << 2
|
(b[0] & 0x1F) << 2
|
||||||
} else {
|
} else {
|
||||||
127
|
127
|
||||||
|
|
Loading…
Reference in New Issue