Compare commits
8 Commits
e79c6c5bfe
...
7c7b75537e
Author | SHA1 | Date |
---|---|---|
Astro | 7c7b75537e | |
Astro | 2847bdc69e | |
Astro | 33c910ff7a | |
Astro | 0bef428894 | |
Astro | 18de86dde2 | |
Astro | daed3f3280 | |
Astro | fa4dabfc11 | |
Astro | 29d5086872 |
File diff suppressed because it is too large
Load Diff
55
flake.lock
55
flake.lock
|
@ -8,11 +8,11 @@
|
||||||
"rust-analyzer-src": "rust-analyzer-src"
|
"rust-analyzer-src": "rust-analyzer-src"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1678515726,
|
"lastModified": 1682317371,
|
||||||
"narHash": "sha256-+WQZ+yfpbyX37fTOPTWrOXAWIjNC65/3SHrSGpbeMUk=",
|
"narHash": "sha256-szJscjPsOR+pFri8eZ6c6PCGw9Ge3llJ2rkz7dyxKwM=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "fenix",
|
"repo": "fenix",
|
||||||
"rev": "1c9e4995fc7022258817f9c9b02c7c4b3d09ca65",
|
"rev": "4ffd0701b8872a8f81ca9492a21fe1c4af17abcf",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -28,11 +28,11 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1671096816,
|
"lastModified": 1679567394,
|
||||||
"narHash": "sha256-ezQCsNgmpUHdZANDCILm3RvtO1xH8uujk/+EqNvzIOg=",
|
"narHash": "sha256-ZvLuzPeARDLiQUt6zSZFGOs+HZmE+3g4QURc8mkBsfM=",
|
||||||
"owner": "nmattia",
|
"owner": "nmattia",
|
||||||
"repo": "naersk",
|
"repo": "naersk",
|
||||||
"rev": "d998160d6a076cfe8f9741e56aeec7e267e3e114",
|
"rev": "88cd22380154a2c36799fe8098888f0f59861a15",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -43,12 +43,11 @@
|
||||||
},
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1678500213,
|
"lastModified": 1680865339,
|
||||||
"narHash": "sha256-A5s2rXawJ+dCThkMXoMuYW8dgyUmkElcyfVJUot/Vr0=",
|
"narHash": "sha256-H6rmJ1CyJ3Q5ZyoLMYq/UEYMS9Q1orJjRpWiQ47HudE=",
|
||||||
"owner": "NixOS",
|
"path": "/nix/store/40w1i37kf1s1ljj3a22ngi5b66bsd225-source",
|
||||||
"repo": "nixpkgs",
|
"rev": "0040164e473509b4aee6aedb3b923e400d6df10b",
|
||||||
"rev": "2ce9b9842b5e63884dfc3dea6689769e2a1ea309",
|
"type": "path"
|
||||||
"type": "github"
|
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"id": "nixpkgs",
|
"id": "nixpkgs",
|
||||||
|
@ -66,11 +65,11 @@
|
||||||
"rust-analyzer-src": {
|
"rust-analyzer-src": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1678443920,
|
"lastModified": 1682284919,
|
||||||
"narHash": "sha256-If9ztMKOJG1jfSAVxbgqnfEqmZHQnfIH5reDvreyfwk=",
|
"narHash": "sha256-Z07/byuJdxLK6E8Yb9qNvUMhUCOWEgYAriojU/wZHu8=",
|
||||||
"owner": "rust-lang",
|
"owner": "rust-lang",
|
||||||
"repo": "rust-analyzer",
|
"repo": "rust-analyzer",
|
||||||
"rev": "9fca0a4afefead3daf8f66fd357999d7cd520880",
|
"rev": "bc78ebd9d83d614562f0a9280bdedf91a3841a73",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -80,13 +79,31 @@
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"utils": {
|
"systems": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1676283394,
|
"lastModified": 1681028828,
|
||||||
"narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=",
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681202837,
|
||||||
|
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||||
"owner": "numtide",
|
"owner": "numtide",
|
||||||
"repo": "flake-utils",
|
"repo": "flake-utils",
|
||||||
"rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073",
|
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|
|
@ -6,6 +6,7 @@ edition = "2021"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
tokio = { version = "1", features = ["full", "tracing"] }
|
tokio = { version = "1", features = ["full", "tracing"] }
|
||||||
|
tokio-uring = "0.4"
|
||||||
reqwest = { version = "0.11", features = ["json", "deflate", "gzip", "trust-dns"] }
|
reqwest = { version = "0.11", features = ["json", "deflate", "gzip", "trust-dns"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
# serde_yaml = "0.9"
|
# serde_yaml = "0.9"
|
||||||
|
|
|
@ -17,8 +17,14 @@ use worker::Message;
|
||||||
#[global_allocator]
|
#[global_allocator]
|
||||||
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||||
|
|
||||||
#[tokio::main]
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
async fn main() {
|
tokio_uring::start(async {
|
||||||
|
run().await;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run() {
|
||||||
cave::systemd::extend_timeout(100_000);
|
cave::systemd::extend_timeout(100_000);
|
||||||
|
|
||||||
cave::init::exit_on_panic();
|
cave::init::exit_on_panic();
|
||||||
|
@ -72,11 +78,9 @@ async fn main() {
|
||||||
.timeout(Duration::from_secs(30))
|
.timeout(Duration::from_secs(30))
|
||||||
.tcp_keepalive(Duration::from_secs(300))
|
.tcp_keepalive(Duration::from_secs(300))
|
||||||
.pool_max_idle_per_host(0)
|
.pool_max_idle_per_host(0)
|
||||||
.user_agent(concat!(
|
.user_agent(
|
||||||
env!("CARGO_PKG_NAME"),
|
format!("{}/{} (+https://fedi.buzz/)", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
|
||||||
"/",
|
)
|
||||||
env!("CARGO_PKG_VERSION"),
|
|
||||||
))
|
|
||||||
.deflate(true)
|
.deflate(true)
|
||||||
.gzip(true)
|
.gzip(true)
|
||||||
.trust_dns(true)
|
.trust_dns(true)
|
||||||
|
|
|
@ -49,8 +49,8 @@ impl Scheduler {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let host = Arc::new(host);
|
let host = Arc::new(host);
|
||||||
|
|
||||||
if self.instances.get(&host).is_none() {
|
if let std::collections::hash_map::Entry::Vacant(entry) = self.instances.entry(host.clone()) {
|
||||||
self.instances.insert(host.clone(), Instance {
|
entry.insert(Instance {
|
||||||
last_fetch: None,
|
last_fetch: None,
|
||||||
error: false,
|
error: false,
|
||||||
});
|
});
|
||||||
|
|
|
@ -2,6 +2,7 @@ use std::collections::HashSet;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
use cave::feed::Post;
|
||||||
use cave::{
|
use cave::{
|
||||||
feed::{Feed, EncodablePost},
|
feed::{Feed, EncodablePost},
|
||||||
store::Store,
|
store::Store,
|
||||||
|
@ -192,25 +193,28 @@ async fn process_posts(
|
||||||
if ! posts_cache.insert(post.uri.clone()) {
|
if ! posts_cache.insert(post.uri.clone()) {
|
||||||
let t1 = Instant::now();
|
let t1 = Instant::now();
|
||||||
|
|
||||||
// introduce instances from reblog authors
|
fn scan_for_hosts(introduce_hosts: &mut HashSet<String>, post: &Post) {
|
||||||
if let Some(reblog_account_host) = post.reblog.as_ref().and_then(|reblog| reblog.account.host()) {
|
// introduce instances from accounts
|
||||||
introduce_hosts.insert(reblog_account_host);
|
if let Some(account_host) = post.account.host() {
|
||||||
}
|
introduce_hosts.insert(account_host);
|
||||||
// introduce instances from mentions
|
}
|
||||||
for mention in &post.mentions {
|
// introduce instances from mentions
|
||||||
if let Some(user_host) = mention.user_host() {
|
for mention in &post.mentions {
|
||||||
introduce_hosts.insert(user_host);
|
if let Some(user_host) = mention.user_host() {
|
||||||
|
introduce_hosts.insert(user_host);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
scan_for_hosts(&mut introduce_hosts, &post);
|
||||||
|
if let Some(reblog) = &post.reblog {
|
||||||
|
scan_for_hosts(&mut introduce_hosts, &reblog);
|
||||||
|
}
|
||||||
|
|
||||||
// check if it's an actual post
|
if let Some(_account_host) = post.account.host() {
|
||||||
if let Some(account_host) = post.account.host() {
|
|
||||||
// send away to redis
|
// send away to redis
|
||||||
if store.save_post(post).await == Ok(true) {
|
if store.save_post(post).await == Ok(true) {
|
||||||
new_posts += 1;
|
new_posts += 1;
|
||||||
}
|
}
|
||||||
// introduce instances from accounts
|
|
||||||
introduce_hosts.insert(account_host);
|
|
||||||
} else {
|
} else {
|
||||||
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
|
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
|
||||||
}
|
}
|
||||||
|
@ -230,10 +234,6 @@ async fn process_posts(
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let introduce_hosts = introduce_hosts.into_iter()
|
|
||||||
.map(|host| host.to_owned())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
(new_post_ratio, introduce_hosts)
|
(new_post_ratio, introduce_hosts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue