From 91fa0714b9b8fa0888960584cadd3742b085f722 Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 12 Oct 2023 21:49:47 +0200 Subject: [PATCH] cave/store, cave/firehose: require redis_password_file --- Cargo.lock | 1 + butcher/src/config.rs | 1 + butcher/src/main.rs | 6 ++++-- cave/Cargo.toml | 1 + cave/src/firehose.rs | 12 +++++++++--- cave/src/store.rs | 13 ++++++++++--- gatherer/src/config.rs | 1 + gatherer/src/main.rs | 6 ++++-- hunter/src/config.rs | 1 + hunter/src/main.rs | 2 +- nixos-module.nix | 8 ++++++++ smokestack/src/config.rs | 1 + smokestack/src/main.rs | 2 +- 13 files changed, 43 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5790d..3f95aab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,6 +379,7 @@ dependencies = [ "tokio-postgres", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/butcher/src/config.rs b/butcher/src/config.rs index 07723de..96a687c 100644 --- a/butcher/src/config.rs +++ b/butcher/src/config.rs @@ -1,5 +1,6 @@ #[derive(Debug, serde::Deserialize)] pub struct Config { pub redis: String, + pub redis_password_file: String, pub profanity: String, } diff --git a/butcher/src/main.rs b/butcher/src/main.rs index e2decb1..fe04e77 100644 --- a/butcher/src/main.rs +++ b/butcher/src/main.rs @@ -40,12 +40,14 @@ async fn main() { let config = config::Config::load(); let profanity = WordList::new(&config.profanity).await; - let store = cave::store::Store::new(16, config.redis.clone()).await; + let store = cave::store::Store::new( + 16, config.redis.clone(), config.redis_password_file.clone() + ).await; cave::systemd::status("Starting trend_setter"); let trend_setter_tx = trend_setter::start(store.clone()); - let firehose_factory = FirehoseFactory::new(config.redis); + let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file); let firehose = firehose_factory.produce() .await .expect("firehose"); diff --git a/cave/Cargo.toml b/cave/Cargo.toml index d7fbb80..17995f2 100644 --- a/cave/Cargo.toml +++ b/cave/Cargo.toml @@ -20,3 +20,4 @@ eventsource-stream = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter"] } inotify = "0.10" tokio-postgres = "0.7" +url = "2" diff --git a/cave/src/firehose.rs b/cave/src/firehose.rs index a5b6da0..396a0c6 100644 --- a/cave/src/firehose.rs +++ b/cave/src/firehose.rs @@ -1,18 +1,24 @@ use futures::{Stream, StreamExt}; use redis::RedisError; +use url::Url; #[derive(Clone)] pub struct FirehoseFactory { - redis_url: String, + redis_url: Url, } impl FirehoseFactory { - pub fn new(redis_url: String) -> Self { + pub fn new(redis_url: String, redis_password_file: String) -> Self { + let redis_password = std::fs::read_to_string(redis_password_file) + .expect("redis_password_file"); + let mut redis_url = Url::parse(&redis_url) + .expect("redis_url"); + redis_url.set_password(Some(&redis_password)).unwrap(); FirehoseFactory { redis_url } } pub async fn produce(&self) -> Result, Vec)>, RedisError> { - let client = redis::Client::open(&self.redis_url[..])?; + let client = redis::Client::open(self.redis_url.clone())?; let mut pubsub_conn = client.get_async_connection() .await? .into_pubsub(); diff --git a/cave/src/store.rs b/cave/src/store.rs index 88afb24..b89f486 100644 --- a/cave/src/store.rs +++ b/cave/src/store.rs @@ -3,6 +3,7 @@ use std::pin::Pin; use bb8::ManageConnection; use futures::{Future, Stream, stream::unfold, StreamExt}; use redis::{Value, RedisError, aio::ConnectionLike, FromRedisValue}; +use url::Url; use crate::{ feed::{EncodablePost, Post}, trend_tag::TrendTag, @@ -22,7 +23,7 @@ pub type Error = RedisError; /// wrapper so we can impl ManageConnection struct RedisPool { - redis_url: String, + redis_url: Url, } impl ManageConnection for RedisPool { @@ -37,7 +38,7 @@ impl ManageConnection for RedisPool { Self: 'async_trait { Box::pin(async { - let client = redis::Client::open(&self.redis_url[..]) + let client = redis::Client::open(self.redis_url.clone()) .expect("redis::Client"); let manager = redis::aio::ConnectionManager::new(client) .await @@ -100,8 +101,14 @@ impl ConnectionLike for Store { } impl Store { - pub async fn new(pool_max_size: u32, redis_url: String) -> Self { + pub async fn new(pool_max_size: u32, redis_url: String, redis_password_file: String) -> Self { crate::systemd::status("Starting redis client"); + let redis_password = std::fs::read_to_string(redis_password_file) + .expect("redis_password_file"); + let mut redis_url = Url::parse(&redis_url) + .expect("redis_url"); + redis_url.set_password(Some(&redis_password)).unwrap(); + let pool = bb8::Pool::builder() .max_size(pool_max_size) .build(RedisPool { redis_url }) diff --git a/gatherer/src/config.rs b/gatherer/src/config.rs index df09a43..4d73796 100644 --- a/gatherer/src/config.rs +++ b/gatherer/src/config.rs @@ -1,6 +1,7 @@ #[derive(Debug, serde::Deserialize)] pub struct Config { pub redis: String, + pub redis_password_file: String, pub database: String, pub listen_port: u16, } diff --git a/gatherer/src/main.rs b/gatherer/src/main.rs index 06a385c..f7f8364 100644 --- a/gatherer/src/main.rs +++ b/gatherer/src/main.rs @@ -39,9 +39,11 @@ async fn main() { cave::systemd::status("Connecting to database"); let db = cave::db::Database::connect(&config.database).await; cave::systemd::status("Starting redis client"); - let store = cave::store::Store::new(8, config.redis.clone()).await; + let store = cave::store::Store::new( + 8, config.redis.clone(), config.redis_password_file.clone() + ).await; - let firehose_factory = FirehoseFactory::new(config.redis); + let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file); let http = http_server::start( config.listen_port, diff --git a/hunter/src/config.rs b/hunter/src/config.rs index 5c7b916..a7d1b5e 100644 --- a/hunter/src/config.rs +++ b/hunter/src/config.rs @@ -1,6 +1,7 @@ #[derive(Debug, serde::Deserialize)] pub struct Config { pub redis: String, + pub redis_password_file: String, pub database: String, pub hosts: Vec, pub max_workers: usize, diff --git a/hunter/src/main.rs b/hunter/src/main.rs index 529dbaf..6457f22 100644 --- a/hunter/src/main.rs +++ b/hunter/src/main.rs @@ -41,7 +41,7 @@ async fn run() { .unwrap(); let db = cave::db::Database::connect(&config.database).await; - let mut store = cave::store::Store::new(16, config.redis).await; + let mut store = cave::store::Store::new(16, config.redis, config.redis_password_file).await; let posts_cache = posts_cache::PostsCache::new(65536); let block_list = block_list::BlockList::new(&config.blocklist).await; diff --git a/nixos-module.nix b/nixos-module.nix index a2c3006..2861107 100644 --- a/nixos-module.nix +++ b/nixos-module.nix @@ -11,6 +11,7 @@ let hunterDefaultSettings = { redis = "redis://127.0.0.1:${toString cfg.redis.port}/"; + redis_password_file = cfg.redis.passwordFile; database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman"; hosts = [ "mastodon.social" ]; max_workers = 16; @@ -26,6 +27,7 @@ let butcherDefaultSettings = { redis = "redis://127.0.0.1:${toString cfg.redis.port}/"; + redis_password_file = cfg.redis.passwordFile; profanity = profanityPath; }; @@ -37,6 +39,7 @@ let gathererDefaultSettings = { redis = "redis://127.0.0.1:${toString cfg.redis.port}/"; + redis_password_file = cfg.redis.passwordFile; database = "host=localhost user=${dbUser} password=${dbPassword} dbname=caveman"; listen_port = 8000; }; @@ -49,6 +52,7 @@ let smokestackDefaultSettings = { redis = "redis://127.0.0.1:${toString cfg.redis.port}/"; + redis_password_file = cfg.redis.passwordFile; listen_port = 23; }; @@ -75,6 +79,9 @@ in type = types.int; default = 8; }; + redis.passwordFile = mkOption { + type = types.path; + }; hunter.enable = mkEnableOption "caveman hunter"; @@ -144,6 +151,7 @@ in services.redis.servers.caveman = { enable = true; port = cfg.redis.port; + passFile = cfg.redis.passwordFile; settings = { inherit (cfg.redis) maxmemory maxmemory-samples; maxmemory-policy = "allkeys-lru"; diff --git a/smokestack/src/config.rs b/smokestack/src/config.rs index d57b322..044537e 100644 --- a/smokestack/src/config.rs +++ b/smokestack/src/config.rs @@ -1,5 +1,6 @@ #[derive(Debug, serde::Deserialize)] pub struct Config { pub redis: String, + pub redis_password_file: String, pub listen_port: u16, } diff --git a/smokestack/src/main.rs b/smokestack/src/main.rs index 9bf4137..99ce193 100644 --- a/smokestack/src/main.rs +++ b/smokestack/src/main.rs @@ -195,7 +195,7 @@ async fn main() { let state = State::new(); - let firehose_factory = FirehoseFactory::new(config.redis); + let firehose_factory = FirehoseFactory::new(config.redis, config.redis_password_file); let firehose = firehose_factory.produce() .await .expect("firehose");