cave/store, cave/firehose: require redis_password_file
This commit is contained in:
parent
d9363e63e2
commit
91fa0714b9
|
@ -379,6 +379,7 @@ dependencies = [
|
||||||
"tokio-postgres",
|
"tokio-postgres",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
#[derive(Debug, serde::Deserialize)]
|
#[derive(Debug, serde::Deserialize)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub redis: String,
|
pub redis: String,
|
||||||
|
pub redis_password_file: String,
|
||||||
pub profanity: String,
|
pub profanity: String,
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,12 +40,14 @@ async fn main() {
|
||||||
let config = config::Config::load();
|
let config = config::Config::load();
|
||||||
let profanity = WordList::new(&config.profanity).await;
|
let profanity = WordList::new(&config.profanity).await;
|
||||||
|
|
||||||
let store = cave::store::Store::new(16, config.redis.clone()).await;
|
let store = cave::store::Store::new(
|
||||||
|
16, config.redis.clone(), config.redis_password_file.clone()
|
||||||
|
).await;
|
||||||
|
|
||||||
cave::systemd::status("Starting trend_setter");
|
cave::systemd::status("Starting trend_setter");
|
||||||
let trend_setter_tx = trend_setter::start(store.clone());
|
let trend_setter_tx = trend_setter::start(store.clone());
|
||||||
|
|
||||||
let firehose_factory = FirehoseFactory::new(config.redis);
|
let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file);
|
||||||
let firehose = firehose_factory.produce()
|
let firehose = firehose_factory.produce()
|
||||||
.await
|
.await
|
||||||
.expect("firehose");
|
.expect("firehose");
|
||||||
|
|
|
@ -20,3 +20,4 @@ eventsource-stream = "0.2"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
inotify = "0.10"
|
inotify = "0.10"
|
||||||
tokio-postgres = "0.7"
|
tokio-postgres = "0.7"
|
||||||
|
url = "2"
|
||||||
|
|
|
@ -1,18 +1,24 @@
|
||||||
use futures::{Stream, StreamExt};
|
use futures::{Stream, StreamExt};
|
||||||
use redis::RedisError;
|
use redis::RedisError;
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct FirehoseFactory {
|
pub struct FirehoseFactory {
|
||||||
redis_url: String,
|
redis_url: Url,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FirehoseFactory {
|
impl FirehoseFactory {
|
||||||
pub fn new(redis_url: String) -> Self {
|
pub fn new(redis_url: String, redis_password_file: String) -> Self {
|
||||||
|
let redis_password = std::fs::read_to_string(redis_password_file)
|
||||||
|
.expect("redis_password_file");
|
||||||
|
let mut redis_url = Url::parse(&redis_url)
|
||||||
|
.expect("redis_url");
|
||||||
|
redis_url.set_password(Some(&redis_password)).unwrap();
|
||||||
FirehoseFactory { redis_url }
|
FirehoseFactory { redis_url }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn produce(&self) -> Result<impl Stream<Item = (Vec<u8>, Vec<u8>)>, RedisError> {
|
pub async fn produce(&self) -> Result<impl Stream<Item = (Vec<u8>, Vec<u8>)>, RedisError> {
|
||||||
let client = redis::Client::open(&self.redis_url[..])?;
|
let client = redis::Client::open(self.redis_url.clone())?;
|
||||||
let mut pubsub_conn = client.get_async_connection()
|
let mut pubsub_conn = client.get_async_connection()
|
||||||
.await?
|
.await?
|
||||||
.into_pubsub();
|
.into_pubsub();
|
||||||
|
|
|
@ -3,6 +3,7 @@ use std::pin::Pin;
|
||||||
use bb8::ManageConnection;
|
use bb8::ManageConnection;
|
||||||
use futures::{Future, Stream, stream::unfold, StreamExt};
|
use futures::{Future, Stream, stream::unfold, StreamExt};
|
||||||
use redis::{Value, RedisError, aio::ConnectionLike, FromRedisValue};
|
use redis::{Value, RedisError, aio::ConnectionLike, FromRedisValue};
|
||||||
|
use url::Url;
|
||||||
use crate::{
|
use crate::{
|
||||||
feed::{EncodablePost, Post},
|
feed::{EncodablePost, Post},
|
||||||
trend_tag::TrendTag,
|
trend_tag::TrendTag,
|
||||||
|
@ -22,7 +23,7 @@ pub type Error = RedisError;
|
||||||
|
|
||||||
/// wrapper so we can impl ManageConnection
|
/// wrapper so we can impl ManageConnection
|
||||||
struct RedisPool {
|
struct RedisPool {
|
||||||
redis_url: String,
|
redis_url: Url,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ManageConnection for RedisPool {
|
impl ManageConnection for RedisPool {
|
||||||
|
@ -37,7 +38,7 @@ impl ManageConnection for RedisPool {
|
||||||
Self: 'async_trait
|
Self: 'async_trait
|
||||||
{
|
{
|
||||||
Box::pin(async {
|
Box::pin(async {
|
||||||
let client = redis::Client::open(&self.redis_url[..])
|
let client = redis::Client::open(self.redis_url.clone())
|
||||||
.expect("redis::Client");
|
.expect("redis::Client");
|
||||||
let manager = redis::aio::ConnectionManager::new(client)
|
let manager = redis::aio::ConnectionManager::new(client)
|
||||||
.await
|
.await
|
||||||
|
@ -100,8 +101,14 @@ impl ConnectionLike for Store {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Store {
|
impl Store {
|
||||||
pub async fn new(pool_max_size: u32, redis_url: String) -> Self {
|
pub async fn new(pool_max_size: u32, redis_url: String, redis_password_file: String) -> Self {
|
||||||
crate::systemd::status("Starting redis client");
|
crate::systemd::status("Starting redis client");
|
||||||
|
let redis_password = std::fs::read_to_string(redis_password_file)
|
||||||
|
.expect("redis_password_file");
|
||||||
|
let mut redis_url = Url::parse(&redis_url)
|
||||||
|
.expect("redis_url");
|
||||||
|
redis_url.set_password(Some(&redis_password)).unwrap();
|
||||||
|
|
||||||
let pool = bb8::Pool::builder()
|
let pool = bb8::Pool::builder()
|
||||||
.max_size(pool_max_size)
|
.max_size(pool_max_size)
|
||||||
.build(RedisPool { redis_url })
|
.build(RedisPool { redis_url })
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#[derive(Debug, serde::Deserialize)]
|
#[derive(Debug, serde::Deserialize)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub redis: String,
|
pub redis: String,
|
||||||
|
pub redis_password_file: String,
|
||||||
pub database: String,
|
pub database: String,
|
||||||
pub listen_port: u16,
|
pub listen_port: u16,
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,9 +39,11 @@ async fn main() {
|
||||||
cave::systemd::status("Connecting to database");
|
cave::systemd::status("Connecting to database");
|
||||||
let db = cave::db::Database::connect(&config.database).await;
|
let db = cave::db::Database::connect(&config.database).await;
|
||||||
cave::systemd::status("Starting redis client");
|
cave::systemd::status("Starting redis client");
|
||||||
let store = cave::store::Store::new(8, config.redis.clone()).await;
|
let store = cave::store::Store::new(
|
||||||
|
8, config.redis.clone(), config.redis_password_file.clone()
|
||||||
|
).await;
|
||||||
|
|
||||||
let firehose_factory = FirehoseFactory::new(config.redis);
|
let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file);
|
||||||
|
|
||||||
let http = http_server::start(
|
let http = http_server::start(
|
||||||
config.listen_port,
|
config.listen_port,
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#[derive(Debug, serde::Deserialize)]
|
#[derive(Debug, serde::Deserialize)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub redis: String,
|
pub redis: String,
|
||||||
|
pub redis_password_file: String,
|
||||||
pub database: String,
|
pub database: String,
|
||||||
pub hosts: Vec<String>,
|
pub hosts: Vec<String>,
|
||||||
pub max_workers: usize,
|
pub max_workers: usize,
|
||||||
|
|
|
@ -41,7 +41,7 @@ async fn run() {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let db = cave::db::Database::connect(&config.database).await;
|
let db = cave::db::Database::connect(&config.database).await;
|
||||||
let mut store = cave::store::Store::new(16, config.redis).await;
|
let mut store = cave::store::Store::new(16, config.redis, config.redis_password_file).await;
|
||||||
let posts_cache = posts_cache::PostsCache::new(65536);
|
let posts_cache = posts_cache::PostsCache::new(65536);
|
||||||
|
|
||||||
let block_list = block_list::BlockList::new(&config.blocklist).await;
|
let block_list = block_list::BlockList::new(&config.blocklist).await;
|
||||||
|
|
|
@ -11,6 +11,7 @@ let
|
||||||
|
|
||||||
hunterDefaultSettings = {
|
hunterDefaultSettings = {
|
||||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||||
|
redis_password_file = cfg.redis.passwordFile;
|
||||||
database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman";
|
database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman";
|
||||||
hosts = [ "mastodon.social" ];
|
hosts = [ "mastodon.social" ];
|
||||||
max_workers = 16;
|
max_workers = 16;
|
||||||
|
@ -26,6 +27,7 @@ let
|
||||||
|
|
||||||
butcherDefaultSettings = {
|
butcherDefaultSettings = {
|
||||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||||
|
redis_password_file = cfg.redis.passwordFile;
|
||||||
profanity = profanityPath;
|
profanity = profanityPath;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -37,6 +39,7 @@ let
|
||||||
|
|
||||||
gathererDefaultSettings = {
|
gathererDefaultSettings = {
|
||||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||||
|
redis_password_file = cfg.redis.passwordFile;
|
||||||
database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman";
|
database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman";
|
||||||
listen_port = 8000;
|
listen_port = 8000;
|
||||||
};
|
};
|
||||||
|
@ -49,6 +52,7 @@ let
|
||||||
|
|
||||||
smokestackDefaultSettings = {
|
smokestackDefaultSettings = {
|
||||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||||
|
redis_password_file = cfg.redis.passwordFile;
|
||||||
listen_port = 23;
|
listen_port = 23;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -75,6 +79,9 @@ in
|
||||||
type = types.int;
|
type = types.int;
|
||||||
default = 8;
|
default = 8;
|
||||||
};
|
};
|
||||||
|
redis.passwordFile = mkOption {
|
||||||
|
type = types.path;
|
||||||
|
};
|
||||||
|
|
||||||
hunter.enable = mkEnableOption "caveman hunter";
|
hunter.enable = mkEnableOption "caveman hunter";
|
||||||
|
|
||||||
|
@ -144,6 +151,7 @@ in
|
||||||
services.redis.servers.caveman = {
|
services.redis.servers.caveman = {
|
||||||
enable = true;
|
enable = true;
|
||||||
port = cfg.redis.port;
|
port = cfg.redis.port;
|
||||||
|
passFile = cfg.redis.passwordFile;
|
||||||
settings = {
|
settings = {
|
||||||
inherit (cfg.redis) maxmemory maxmemory-samples;
|
inherit (cfg.redis) maxmemory maxmemory-samples;
|
||||||
maxmemory-policy = "allkeys-lru";
|
maxmemory-policy = "allkeys-lru";
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
#[derive(Debug, serde::Deserialize)]
|
#[derive(Debug, serde::Deserialize)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub redis: String,
|
pub redis: String,
|
||||||
|
pub redis_password_file: String,
|
||||||
pub listen_port: u16,
|
pub listen_port: u16,
|
||||||
}
|
}
|
||||||
|
|
|
@ -195,7 +195,7 @@ async fn main() {
|
||||||
|
|
||||||
let state = State::new();
|
let state = State::new();
|
||||||
|
|
||||||
let firehose_factory = FirehoseFactory::new(config.redis);
|
let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file);
|
||||||
let firehose = firehose_factory.produce()
|
let firehose = firehose_factory.produce()
|
||||||
.await
|
.await
|
||||||
.expect("firehose");
|
.expect("firehose");
|
||||||
|
|
Loading…
Reference in New Issue