Compare commits

...

4 Commits

Author SHA1 Message Date
Astro 527f13efd5 gatherer/templates/token_donate: embolden 2023-08-11 21:04:17 +02:00
Astro 91b1edcad4 cargo update 2023-08-11 21:03:39 +02:00
Astro 93d01f4233 butcher: remove useless tag_trimmer 2023-08-11 21:00:32 +02:00
Astro a871723e9b flake.lock: Update
Flake lock file updates:

• Updated input 'fenix':
    'github:nix-community/fenix/5dc84891da35116d088496499292eebb9cd2f88b' (2023-07-23)
  → 'github:nix-community/fenix/492f9d57e90e53e09af6e3ada26c866af0cf4bf0' (2023-08-11)
• Updated input 'fenix/rust-analyzer-src':
    'github:rust-lang/rust-analyzer/99718d0c8bc5aadd993acdcabc1778fc7b5cc572' (2023-07-22)
  → 'github:rust-lang/rust-analyzer/1b678231d71f48f078e1a80230c28a2fce2daec5' (2023-08-10)
• Updated input 'naersk':
    'github:nmattia/naersk/abca1fb7a6cfdd355231fc220c3d0302dbb4369a' (2023-07-05)
  → 'github:nmattia/naersk/d9a33d69a9c421d64c8d925428864e93be895dcc' (2023-07-26)
• Updated input 'nixpkgs':
    'path:/nix/store/210f1wyfbyg648j1sqvxcwa29nj61db5-source?lastModified=1689679375&narHash=sha256-LHUC52WvyVDi9PwyL1QCpaxYWBqp4ir4iL6zgOkmcb8%3D&rev=684c17c429c42515bafb3ad775d2a710947f3d67' (2023-07-18)
  → 'path:/nix/store/9xpvg56sk83p217x1ws6a42ga07mip2d-source?lastModified=1690640159&narHash=sha256-5DZUYnkeMOsVb/eqPYb9zns5YsnQXRJRC8Xx/nPMcno%3D&rev=e6ab46982debeab9831236869539a507f670a129' (2023-07-29)
2023-08-11 20:57:15 +02:00
6 changed files with 80 additions and 240 deletions

130
Cargo.lock generated
View File

@ -39,9 +39,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c"
dependencies = [
"memchr",
]
@ -151,7 +151,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -280,9 +280,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.3.3"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "block-buffer"
@ -430,9 +430,12 @@ dependencies = [
[[package]]
name = "cc"
version = "1.0.79"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
dependencies = [
"libc",
]
[[package]]
name = "cfg-if"
@ -597,9 +600,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.1"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
dependencies = [
"errno-dragonfly",
"libc",
@ -682,7 +685,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -762,7 +765,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -1037,9 +1040,9 @@ dependencies = [
[[package]]
name = "inotify"
version = "0.10.1"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff335215fb898bf09c45833b657233d8c0b699a616d7dd64d0513080da270ab6"
checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc"
dependencies = [
"bitflags 1.3.2",
"futures-core",
@ -1093,9 +1096,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "jemalloc-sys"
version = "0.5.3+5.3.0-patched"
version = "0.5.4+5.3.0-patched"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9bd5d616ea7ed58b571b2e209a65759664d7fb021a0819d7a790afc67e47ca1"
checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2"
dependencies = [
"cc",
"libc",
@ -1103,9 +1106,9 @@ dependencies = [
[[package]]
name = "jemallocator"
version = "0.5.0"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6"
checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc"
dependencies = [
"jemalloc-sys",
"libc",
@ -1151,9 +1154,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.4.3"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
[[package]]
name = "lock_api"
@ -1427,9 +1430,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "openssl"
version = "0.10.55"
version = "0.10.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
dependencies = [
"bitflags 1.3.2",
"cfg-if",
@ -1448,7 +1451,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -1459,9 +1462,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.90"
version = "0.9.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
dependencies = [
"cc",
"libc",
@ -1533,29 +1536,29 @@ dependencies = [
[[package]]
name = "pin-project"
version = "1.1.2"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.2"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
name = "pin-project-lite"
version = "0.2.10"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
[[package]]
name = "pin-utils"
@ -1575,14 +1578,14 @@ version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e"
dependencies = [
"portable-atomic 1.4.1",
"portable-atomic 1.4.2",
]
[[package]]
name = "portable-atomic"
version = "1.4.1"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6"
checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e"
[[package]]
name = "postgres-protocol"
@ -1741,13 +1744,13 @@ dependencies = [
[[package]]
name = "regex"
version = "1.9.1"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
dependencies = [
"aho-corasick 1.0.2",
"aho-corasick 1.0.3",
"memchr",
"regex-automata 0.3.3",
"regex-automata 0.3.6",
"regex-syntax 0.7.4",
]
@ -1762,11 +1765,11 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
dependencies = [
"aho-corasick 1.0.2",
"aho-corasick 1.0.3",
"memchr",
"regex-syntax 0.7.4",
]
@ -1842,11 +1845,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustix"
version = "0.38.4"
version = "0.38.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
dependencies = [
"bitflags 2.3.3",
"bitflags 2.4.0",
"errno",
"libc",
"linux-raw-sys",
@ -1905,29 +1908,29 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.174"
version = "1.0.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1"
checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.174"
version = "1.0.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e"
checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
name = "serde_json"
version = "1.0.103"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
dependencies = [
"itoa",
"ryu",
@ -2070,9 +2073,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.27"
version = "2.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
dependencies = [
"proc-macro2",
"quote",
@ -2102,9 +2105,9 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.7.0"
version = "3.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
dependencies = [
"cfg-if",
"fastrand",
@ -2146,7 +2149,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -2186,11 +2189,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.29.1"
version = "1.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd"
dependencies = [
"autocfg",
"backtrace",
"bytes",
"libc",
@ -2199,7 +2201,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.4.9",
"socket2 0.5.3",
"tokio-macros",
"tracing",
"windows-sys",
@ -2213,7 +2215,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -2361,7 +2363,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -2579,7 +2581,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
"wasm-bindgen-shared",
]
@ -2613,7 +2615,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]

View File

@ -13,7 +13,6 @@ use trend_setter::UpdateSet;
mod config;
mod trend_setter;
mod tag_trimmer;
async fn is_profane(profanity: &WordList, post: &Post) -> bool {
if post.sensitive == Some(true) {
@ -43,9 +42,6 @@ async fn main() {
cave::systemd::status("Starting trend_setter");
let trend_setter_tx = trend_setter::start(store.clone());
cave::systemd::status("Starting tag_trimmer");
tag_trimmer::start(store.clone());
let firehose_factory = FirehoseFactory::new(config.redis);
let firehose = firehose_factory.produce()
.await

View File

@ -1,79 +0,0 @@
use std::time::{Duration, Instant};
use futures::{
future::join,
StreamExt,
};
use tokio::time::sleep;
use cave::store::Store;
const TRIM_INTERVAL: Duration = Duration::from_secs(86400);
const TRIM_STEP_SLEEP: Duration = Duration::from_millis(30);
pub fn start(store: Store) {
tokio::spawn(async move {
loop {
let start = Instant::now();
run(&store).await;
let end = Instant::now();
tracing::info!("trimmed in {:.3?}", end - start);
sleep(TRIM_INTERVAL).await;
}
});
}
async fn run(store: &Store) {
let mut store1 = store.clone();
let mut store3 = store.clone();
join(async {
let store2 = store1.clone();
store1.get_tags_global()
.await
.unwrap()
.for_each(move |tag| {
let mut store2 = store2.clone();
async move {
trim_tag(None, tag, &mut store2).await;
sleep(TRIM_STEP_SLEEP).await;
}
})
.await
}, async {
let store4 = store3.clone();
store3.get_tags_by_language()
.await
.unwrap()
.for_each(move |(language, tag)| {
let mut store4 = store4.clone();
async move {
trim_tag(language, tag, &mut store4).await;
sleep(TRIM_STEP_SLEEP).await;
}})
.await
}).await;
}
async fn trim_tag(language: Option<String>, tag: String, store: &mut Store) {
let t1 = Instant::now();
let lang = if language.is_some() { "some" } else { "any" };
let trend_tags = store.get_trend_tags(&language, [tag].into_iter())
.await
.unwrap();
let trend_tag = &trend_tags[0];
if trend_tag.hour_users.iter().all(|(_, users)| *users == 0) {
// drop the whole wholly-outdated tag
tracing::debug!("Deleting whole tag {:?} {}", language, trend_tag.name);
store.delete_tag(&language, &trend_tag.name).await
.expect("delete_tag");
metrics::increment_counter!("hunter_trimmed_tags", "type" => "delete_tag", "lang" => lang);
} else {
store.clean_trend_tag(&language, trend_tag).await
.expect("clean_trend_tag");
metrics::increment_counter!("hunter_trimmed_tags", "type" => "clean_trend_tag", "lang" => lang);
}
let t2 = Instant::now();
metrics::histogram!("hunter_trim_tag_seconds", t2 - t1, "lang" => lang);
}

View File

@ -8,11 +8,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1690093235,
"narHash": "sha256-78rST3nYLd0DxWo1l0ZqHoQ+jI0bkLp+pAElC/S01rU=",
"lastModified": 1691734822,
"narHash": "sha256-mIK3x93yNVB0IkwaO3nQ1au0sVBo8QiLf5mpOx88hZ0=",
"owner": "nix-community",
"repo": "fenix",
"rev": "5dc84891da35116d088496499292eebb9cd2f88b",
"rev": "492f9d57e90e53e09af6e3ada26c866af0cf4bf0",
"type": "github"
},
"original": {
@ -28,11 +28,11 @@
]
},
"locked": {
"lastModified": 1688534083,
"narHash": "sha256-/bI5vsioXscQTsx+Hk9X5HfweeNZz/6kVKsbdqfwW7g=",
"lastModified": 1690373729,
"narHash": "sha256-e136hTT7LqQ2QjOTZQMW+jnsevWwBpMj78u6FRUsH9I=",
"owner": "nmattia",
"repo": "naersk",
"rev": "abca1fb7a6cfdd355231fc220c3d0302dbb4369a",
"rev": "d9a33d69a9c421d64c8d925428864e93be895dcc",
"type": "github"
},
"original": {
@ -43,10 +43,10 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1689679375,
"narHash": "sha256-LHUC52WvyVDi9PwyL1QCpaxYWBqp4ir4iL6zgOkmcb8=",
"path": "/nix/store/210f1wyfbyg648j1sqvxcwa29nj61db5-source",
"rev": "684c17c429c42515bafb3ad775d2a710947f3d67",
"lastModified": 1690640159,
"narHash": "sha256-5DZUYnkeMOsVb/eqPYb9zns5YsnQXRJRC8Xx/nPMcno=",
"path": "/nix/store/9xpvg56sk83p217x1ws6a42ga07mip2d-source",
"rev": "e6ab46982debeab9831236869539a507f670a129",
"type": "path"
},
"original": {
@ -65,11 +65,11 @@
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1690057540,
"narHash": "sha256-MKGhZsFTpJH3Sq+9dGFGqOje3A6PD6fKGO92tM23zuY=",
"lastModified": 1691691861,
"narHash": "sha256-EBNtN+r7eIrNrZTpPHwCV/KpEqd36ZyDuf8aUyiCFOU=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "99718d0c8bc5aadd993acdcabc1778fc7b5cc572",
"rev": "1b678231d71f48f078e1a80230c28a2fce2daec5",
"type": "github"
},
"original": {

View File

@ -44,7 +44,7 @@
<h2>Update: Token permissions</h2>
<p>
Since the introduction of this functionality, we have been made aware that the <em>reading statuses</em> permission <b>allows to your read private messages</b> - just like what your Mastodon admins can see. It is unfathomable why anyone would communicate privately without end-to-end encryption in 2023 but if you still do: <em>Consider creating an empty dummy account on your instance for the token!</em>
Since the introduction of this functionality, we have been made aware that the <em>reading statuses</em> permission <b>allows to your read private messages</b> - just like what your Mastodon admins can see. It is unfathomable why anyone would communicate privately without end-to-end encryption in 2023 but if you still do: <b>Consider creating an empty dummy account on your instance for the token!</b>
</p>
</main>

View File

@ -1,79 +0,0 @@
use std::time::{Duration, Instant};
use futures::{
future::join,
StreamExt,
};
use tokio::time::sleep;
use cave::store::Store;
const TRIM_INTERVAL: Duration = Duration::from_secs(86400);
const TRIM_STEP_SLEEP: Duration = Duration::from_millis(30);
pub fn start(store: Store) {
tokio::spawn(async move {
loop {
let start = Instant::now();
run(&store).await;
let end = Instant::now();
tracing::info!("trimmed in {:.3?}", end - start);
sleep(TRIM_INTERVAL).await;
}
});
}
async fn run(store: &Store) {
let mut store1 = store.clone();
let mut store3 = store.clone();
join(async {
let store2 = store1.clone();
store1.get_tags_global()
.await
.unwrap()
.for_each(move |tag| {
let mut store2 = store2.clone();
async move {
trim_tag(None, tag, &mut store2).await;
sleep(TRIM_STEP_SLEEP).await;
}
})
.await
}, async {
let store4 = store3.clone();
store3.get_tags_by_language()
.await
.unwrap()
.for_each(move |(language, tag)| {
let mut store4 = store4.clone();
async move {
trim_tag(language, tag, &mut store4).await;
sleep(TRIM_STEP_SLEEP).await;
}})
.await
}).await;
}
async fn trim_tag(language: Option<String>, tag: String, store: &mut Store) {
let t1 = Instant::now();
let lang = if language.is_some() { "some" } else { "any" };
let trend_tags = store.get_trend_tags(&language, [tag].into_iter())
.await
.unwrap();
let trend_tag = &trend_tags[0];
if trend_tag.hour_users.iter().all(|(_, users)| *users == 0) {
// drop the whole wholly-outdated tag
tracing::debug!("Deleting whole tag {:?} {}", language, trend_tag.name);
store.delete_tag(&language, &trend_tag.name).await
.expect("delete_tag");
metrics::increment_counter!("hunter_trimmed_tags", "type" => "delete_tag", "lang" => lang);
} else {
store.clean_trend_tag(&language, trend_tag).await
.expect("clean_trend_tag");
metrics::increment_counter!("hunter_trimmed_tags", "type" => "clean_trend_tag", "lang" => lang);
}
let t2 = Instant::now();
metrics::histogram!("hunter_trim_tag_seconds", t2 - t1, "lang" => lang);
}