Compare commits
8 Commits
e79c6c5bfe
...
7c7b75537e
Author | SHA1 | Date |
---|---|---|
Astro | 7c7b75537e | |
Astro | 2847bdc69e | |
Astro | 33c910ff7a | |
Astro | 0bef428894 | |
Astro | 18de86dde2 | |
Astro | daed3f3280 | |
Astro | fa4dabfc11 | |
Astro | 29d5086872 |
File diff suppressed because it is too large
Load Diff
55
flake.lock
55
flake.lock
|
@ -8,11 +8,11 @@
|
|||
"rust-analyzer-src": "rust-analyzer-src"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1678515726,
|
||||
"narHash": "sha256-+WQZ+yfpbyX37fTOPTWrOXAWIjNC65/3SHrSGpbeMUk=",
|
||||
"lastModified": 1682317371,
|
||||
"narHash": "sha256-szJscjPsOR+pFri8eZ6c6PCGw9Ge3llJ2rkz7dyxKwM=",
|
||||
"owner": "nix-community",
|
||||
"repo": "fenix",
|
||||
"rev": "1c9e4995fc7022258817f9c9b02c7c4b3d09ca65",
|
||||
"rev": "4ffd0701b8872a8f81ca9492a21fe1c4af17abcf",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -28,11 +28,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1671096816,
|
||||
"narHash": "sha256-ezQCsNgmpUHdZANDCILm3RvtO1xH8uujk/+EqNvzIOg=",
|
||||
"lastModified": 1679567394,
|
||||
"narHash": "sha256-ZvLuzPeARDLiQUt6zSZFGOs+HZmE+3g4QURc8mkBsfM=",
|
||||
"owner": "nmattia",
|
||||
"repo": "naersk",
|
||||
"rev": "d998160d6a076cfe8f9741e56aeec7e267e3e114",
|
||||
"rev": "88cd22380154a2c36799fe8098888f0f59861a15",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -43,12 +43,11 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1678500213,
|
||||
"narHash": "sha256-A5s2rXawJ+dCThkMXoMuYW8dgyUmkElcyfVJUot/Vr0=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "2ce9b9842b5e63884dfc3dea6689769e2a1ea309",
|
||||
"type": "github"
|
||||
"lastModified": 1680865339,
|
||||
"narHash": "sha256-H6rmJ1CyJ3Q5ZyoLMYq/UEYMS9Q1orJjRpWiQ47HudE=",
|
||||
"path": "/nix/store/40w1i37kf1s1ljj3a22ngi5b66bsd225-source",
|
||||
"rev": "0040164e473509b4aee6aedb3b923e400d6df10b",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
|
@ -66,11 +65,11 @@
|
|||
"rust-analyzer-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1678443920,
|
||||
"narHash": "sha256-If9ztMKOJG1jfSAVxbgqnfEqmZHQnfIH5reDvreyfwk=",
|
||||
"lastModified": 1682284919,
|
||||
"narHash": "sha256-Z07/byuJdxLK6E8Yb9qNvUMhUCOWEgYAriojU/wZHu8=",
|
||||
"owner": "rust-lang",
|
||||
"repo": "rust-analyzer",
|
||||
"rev": "9fca0a4afefead3daf8f66fd357999d7cd520880",
|
||||
"rev": "bc78ebd9d83d614562f0a9280bdedf91a3841a73",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -80,13 +79,31 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"utils": {
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1676283394,
|
||||
"narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=",
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
@ -6,6 +6,7 @@ edition = "2021"
|
|||
[dependencies]
|
||||
futures = "0.3"
|
||||
tokio = { version = "1", features = ["full", "tracing"] }
|
||||
tokio-uring = "0.4"
|
||||
reqwest = { version = "0.11", features = ["json", "deflate", "gzip", "trust-dns"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
# serde_yaml = "0.9"
|
||||
|
|
|
@ -17,8 +17,14 @@ use worker::Message;
|
|||
#[global_allocator]
|
||||
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tokio_uring::start(async {
|
||||
run().await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
async fn run() {
|
||||
cave::systemd::extend_timeout(100_000);
|
||||
|
||||
cave::init::exit_on_panic();
|
||||
|
@ -72,11 +78,9 @@ async fn main() {
|
|||
.timeout(Duration::from_secs(30))
|
||||
.tcp_keepalive(Duration::from_secs(300))
|
||||
.pool_max_idle_per_host(0)
|
||||
.user_agent(concat!(
|
||||
env!("CARGO_PKG_NAME"),
|
||||
"/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
))
|
||||
.user_agent(
|
||||
format!("{}/{} (+https://fedi.buzz/)", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
|
||||
)
|
||||
.deflate(true)
|
||||
.gzip(true)
|
||||
.trust_dns(true)
|
||||
|
|
|
@ -49,8 +49,8 @@ impl Scheduler {
|
|||
let now = Instant::now();
|
||||
let host = Arc::new(host);
|
||||
|
||||
if self.instances.get(&host).is_none() {
|
||||
self.instances.insert(host.clone(), Instance {
|
||||
if let std::collections::hash_map::Entry::Vacant(entry) = self.instances.entry(host.clone()) {
|
||||
entry.insert(Instance {
|
||||
last_fetch: None,
|
||||
error: false,
|
||||
});
|
||||
|
|
|
@ -2,6 +2,7 @@ use std::collections::HashSet;
|
|||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use cave::feed::Post;
|
||||
use cave::{
|
||||
feed::{Feed, EncodablePost},
|
||||
store::Store,
|
||||
|
@ -192,25 +193,28 @@ async fn process_posts(
|
|||
if ! posts_cache.insert(post.uri.clone()) {
|
||||
let t1 = Instant::now();
|
||||
|
||||
// introduce instances from reblog authors
|
||||
if let Some(reblog_account_host) = post.reblog.as_ref().and_then(|reblog| reblog.account.host()) {
|
||||
introduce_hosts.insert(reblog_account_host);
|
||||
}
|
||||
// introduce instances from mentions
|
||||
for mention in &post.mentions {
|
||||
if let Some(user_host) = mention.user_host() {
|
||||
introduce_hosts.insert(user_host);
|
||||
fn scan_for_hosts(introduce_hosts: &mut HashSet<String>, post: &Post) {
|
||||
// introduce instances from accounts
|
||||
if let Some(account_host) = post.account.host() {
|
||||
introduce_hosts.insert(account_host);
|
||||
}
|
||||
// introduce instances from mentions
|
||||
for mention in &post.mentions {
|
||||
if let Some(user_host) = mention.user_host() {
|
||||
introduce_hosts.insert(user_host);
|
||||
}
|
||||
}
|
||||
}
|
||||
scan_for_hosts(&mut introduce_hosts, &post);
|
||||
if let Some(reblog) = &post.reblog {
|
||||
scan_for_hosts(&mut introduce_hosts, &reblog);
|
||||
}
|
||||
|
||||
// check if it's an actual post
|
||||
if let Some(account_host) = post.account.host() {
|
||||
if let Some(_account_host) = post.account.host() {
|
||||
// send away to redis
|
||||
if store.save_post(post).await == Ok(true) {
|
||||
new_posts += 1;
|
||||
}
|
||||
// introduce instances from accounts
|
||||
introduce_hosts.insert(account_host);
|
||||
} else {
|
||||
tracing::warn!("drop repost ({:?} on {})", post.account.host(), host);
|
||||
}
|
||||
|
@ -230,10 +234,6 @@ async fn process_posts(
|
|||
None
|
||||
};
|
||||
|
||||
let introduce_hosts = introduce_hosts.into_iter()
|
||||
.map(|host| host.to_owned())
|
||||
.collect();
|
||||
|
||||
(new_post_ratio, introduce_hosts)
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue