cave/store: gather 5 images per tag
This commit is contained in:
parent
f03a83e9cd
commit
3e2efe9cf1
|
@ -51,17 +51,29 @@ impl Mention {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct MediaAttachment {
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
pub media_type: String,
|
||||||
|
pub remote_url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||||
pub struct Post {
|
pub struct Post {
|
||||||
pub created_at: String,
|
pub created_at: String,
|
||||||
pub uri: String,
|
pub uri: String,
|
||||||
|
#[serde(default = "String::new")]
|
||||||
pub content: String,
|
pub content: String,
|
||||||
pub account: Account,
|
pub account: Account,
|
||||||
|
#[serde(default)]
|
||||||
pub tags: Vec<Tag>,
|
pub tags: Vec<Tag>,
|
||||||
pub application: Option<Application>,
|
pub application: Option<Application>,
|
||||||
pub sensitive: Option<bool>,
|
pub sensitive: Option<bool>,
|
||||||
|
#[serde(default)]
|
||||||
pub mentions: Vec<Mention>,
|
pub mentions: Vec<Mention>,
|
||||||
pub language: Option<String>,
|
pub language: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub media_attachments: Vec<MediaAttachment>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Post {
|
impl Post {
|
||||||
|
|
|
@ -9,6 +9,7 @@ const POST_EXPIRE: usize = 86400;
|
||||||
const TAG_EXPIRE: u64 = 30 * 24;
|
const TAG_EXPIRE: u64 = 30 * 24;
|
||||||
|
|
||||||
pub const TREND_POOL_SIZE: usize = 20;
|
pub const TREND_POOL_SIZE: usize = 20;
|
||||||
|
pub const IMAGES_PER_TAG: usize = 5;
|
||||||
|
|
||||||
pub type Error = RedisError;
|
pub type Error = RedisError;
|
||||||
|
|
||||||
|
@ -168,7 +169,7 @@ impl Store {
|
||||||
let language = post.lang();
|
let language = post.lang();
|
||||||
|
|
||||||
let mut cmd = redis::pipe();
|
let mut cmd = redis::pipe();
|
||||||
let mut store_tags = |spellings, tag_key, user_key| {
|
let store_tags = |cmd: &mut redis::Pipeline, spellings, tag_key, user_key| {
|
||||||
// by spelling
|
// by spelling
|
||||||
for spelling in spellings {
|
for spelling in spellings {
|
||||||
cmd.hincr(
|
cmd.hincr(
|
||||||
|
@ -186,29 +187,62 @@ impl Store {
|
||||||
if let Some(user_id) = &user_id {
|
if let Some(user_id) = &user_id {
|
||||||
// users by tag/hour
|
// users by tag/hour
|
||||||
cmd.sadd(&user_key, &user_id).ignore()
|
cmd.sadd(&user_key, &user_id).ignore()
|
||||||
.expire(&user_key, TAG_EXPIRE as usize * 86400);
|
.expire(&user_key, TAG_EXPIRE as usize * 86400)
|
||||||
|
.ignore();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let images = if post.sensitive == Some(false) {
|
||||||
|
post.media_attachments.iter()
|
||||||
|
.filter(|a| a.media_type == "image")
|
||||||
|
.filter_map(|a| a.remote_url.as_ref())
|
||||||
|
.take(2)
|
||||||
|
.collect::<Vec<&String>>()
|
||||||
|
} else {
|
||||||
|
// ignore disturbing porn images from sensitive posts
|
||||||
|
vec![]
|
||||||
|
};
|
||||||
|
let mut image_keys = vec![];
|
||||||
for (name, spellings) in post.tags_set() {
|
for (name, spellings) in post.tags_set() {
|
||||||
log::debug!("tag {}", name);
|
log::debug!("tag {}", name);
|
||||||
// global
|
// global
|
||||||
store_tags(
|
store_tags(&mut cmd,
|
||||||
spellings.clone(),
|
spellings.clone(),
|
||||||
format!("g:{}", name),
|
format!("g:{}", name),
|
||||||
format!("u::{}:{}", hour, name),
|
format!("u::{}:{}", hour, name),
|
||||||
);
|
);
|
||||||
// by language
|
// by language
|
||||||
if let Some(language) = &language {
|
if let Some(language) = &language {
|
||||||
store_tags(
|
store_tags(&mut cmd,
|
||||||
spellings,
|
spellings,
|
||||||
format!("l:{}:{}", language, name),
|
format!("l:{}:{}", language, name),
|
||||||
format!("u:{}:{}:{}", language, hour, name),
|
format!("u:{}:{}:{}", language, hour, name),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for image in &images {
|
||||||
|
let image_key = format!("i:{}", name);
|
||||||
|
cmd.sadd(&image_key, image)
|
||||||
|
.ignore()
|
||||||
|
.expire(&image_key, TAG_EXPIRE as usize * 86400)
|
||||||
|
.ignore()
|
||||||
|
.scard(&image_key);
|
||||||
|
image_keys.push(image_key);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match cmd.query_async(self).await {
|
match cmd.query_async::<_, Vec<usize>>(self).await {
|
||||||
Ok(()) => {}
|
Ok(image_key_sizes) => {
|
||||||
|
assert_eq!(image_keys.len(), image_key_sizes.len());
|
||||||
|
let mut cmd = redis::pipe();
|
||||||
|
for (image_key, size) in image_keys.into_iter().zip(image_key_sizes.into_iter()) {
|
||||||
|
let excess = size.saturating_sub(IMAGES_PER_TAG);
|
||||||
|
if excess > 0 {
|
||||||
|
cmd.spop(image_key).arg(excess)
|
||||||
|
.ignore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = cmd.query_async::<_, ()>(self).await;
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("redis error: {:?}", e);
|
log::error!("redis error: {:?}", e);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user