astro/caveman
astro/caveman
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make languages depend on trend pool size

Browse Source
This commit is contained in:
Astro 2022-11-11 17:11:13 +01:00
parent 8a06aa10d5
commit 51a4f7c5ea
3 changed files with 28 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cave/src/store.rs
View File

@ -163,10 +163,6 @@ impl Store {
let language = post.lang();
let mut cmd = redis::pipe();
if let Some(language) = &language {
cmd.hincr("r", language, 1)
.ignore();
}
let mut store_tags = |spellings, tag_key| {
// by hour
cmd.hincr(
@ -219,22 +215,10 @@ impl Store {
Ok(results)
}
pub async fn get_languages(&mut self) -> Result<Vec<(String, usize)>, RedisError> {
let values: Vec<String> = redis::Cmd::hgetall("r")
pub async fn get_languages(&mut self) -> Result<Vec<String>, RedisError> {
redis::Cmd::hkeys("r")
.query_async(self)
.await?;
let mut results = Vec::with_capacity(values.len() / 2);
let mut key = None;
for value in values {
if let Some(key) = key.take() {
if let Ok(count) = str::parse(&value) {
results.push((key, count));
}
} else {
key = Some(value);
}
}
Ok(results)
.await
}
async fn scan<F: FnMut(String)>(
@ -295,11 +279,13 @@ impl Store {
pub async fn update_trend_pools(
&mut self,
remove: impl Iterator<Item = (&Option<String>, u64, Vec<&str>)>,
add: impl Iterator<Item = (&Option<String>, u64, Vec<&str>)>,
language: &Option<String>,
remove: impl Iterator<Item = (u64, Vec<&str>)>,
add: impl Iterator<Item = (u64, Vec<&str>)>,
pool_sizes: impl Iterator<Item = (u64, usize)>,
) -> Result<(), RedisError> {
let mut cmd = redis::pipe();
for (language, period, tags) in remove {
for (period, tags) in remove {
if ! tags.is_empty() {
let pool_key = pool_key(language, period);
cmd.srem(&pool_key, tags)
@ -308,7 +294,7 @@ impl Store {
.ignore();
}
}
for (language, period, tags) in add {
for (period, tags) in add {
if ! tags.is_empty() {
let pool_key = pool_key(language, period);
cmd.sadd(&pool_key, tags)
@ -317,6 +303,18 @@ impl Store {
.ignore();
}
}
if let Some(language) = language {
let max_pool_size = pool_sizes.map(|(_, pool_size)| pool_size)
.max()
.unwrap_or(0);
if max_pool_size > 0 {
cmd.hset("r", language, max_pool_size)
.ignore();
} else {
cmd.hdel("r", language)
.ignore();
}
}
cmd.query_async(self)
.await?;
Ok(())

5
gatherer/src/main.rs
View File

@ -27,11 +27,6 @@ impl Muxable for RequestFactory {
let results = trends::TrendAnalyzer::run(&mut store, 20, &[4, 24, 24 * 7], language)
.await.unwrap();
let mut languages = store.get_languages().await.unwrap();
languages.sort_by(|(_, count1), (_, count2)| count2.cmp(count1));
languages.truncate(23.min(languages.len()));
let mut languages = languages.into_iter()
.map(|(l, _)| l)
.collect::<Vec<_>>();
languages.sort();
(results, languages)
}

12
hunter/src/trend_setter.rs
View File

@ -163,20 +163,22 @@ async fn run(
.filter(|tag| old_tags.get(*tag).is_none())
.collect::<Vec<&str>>();
(period1, scores.len(), remove, add)
(period1, keep, scores.len(), remove, add)
});
// print
for (period, total, remove, add) in updates.clone() {
for (period, keep, total, remove, add) in updates.clone() {
if add.len() > 0 || remove.len() > 0 {
log::info!("Trending in {:?} for {} of {}: +{:?} -{:?}", language, period, total, add, remove);
log::info!("Trending in {:?} for {} of {}/{}: +{:?} -{:?}", language, period, keep, total, add, remove);
}
}
// write to redis
store.update_trend_pools(
updates.clone().map(|(period, _, remove, _add)| (language, period, remove)),
updates.map(|(period, _, _remove, add)| (language, period, add)),
language,
updates.clone().map(|(period, _, _, remove, _add)| (period, remove)),
updates.clone().map(|(period, _, _, _remove, add)| (period, add)),
updates.map(|(period, keep, _, _, _)| (period, keep)),
).await?;
Ok(())