begin collecting tokens
This commit is contained in:
parent
5839a02e55
commit
aa1d0f1009
|
|
@ -284,6 +284,15 @@ version = "2.3.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "0.2.17"
|
||||
|
|
@ -307,6 +316,12 @@ version = "3.13.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.4.0"
|
||||
|
|
@ -329,6 +344,7 @@ dependencies = [
|
|||
"serde_yaml",
|
||||
"systemd",
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
|
@ -369,10 +385,12 @@ dependencies = [
|
|||
"metrics-exporter-prometheus",
|
||||
"metrics-util",
|
||||
"redis",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_yaml",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -394,6 +412,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tokio-uring",
|
||||
"tracing",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -466,6 +485,15 @@ version = "0.8.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.3.2"
|
||||
|
|
@ -497,6 +525,16 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cstr-argument"
|
||||
version = "0.1.2"
|
||||
|
|
@ -513,6 +551,17 @@ version = "2.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.32"
|
||||
|
|
@ -578,6 +627,12 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.0.0"
|
||||
|
|
@ -740,6 +795,16 @@ dependencies = [
|
|||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.10"
|
||||
|
|
@ -803,6 +868,15 @@ version = "0.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hostname"
|
||||
version = "0.3.1"
|
||||
|
|
@ -1142,6 +1216,15 @@ version = "0.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
|
|
@ -1430,6 +1513,24 @@ version = "2.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.1.2"
|
||||
|
|
@ -1483,6 +1584,35 @@ version = "1.4.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6"
|
||||
|
||||
[[package]]
|
||||
name = "postgres-protocol"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"hmac",
|
||||
"md-5",
|
||||
"memchr",
|
||||
"rand",
|
||||
"sha2",
|
||||
"stringprep",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postgres-types"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"postgres-protocol",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -1835,6 +1965,17 @@ version = "1.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.4"
|
||||
|
|
@ -1853,6 +1994,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.2.1"
|
||||
|
|
@ -1894,6 +2041,22 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
|
@ -2063,6 +2226,30 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-postgres"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"log",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"phf",
|
||||
"pin-project-lite",
|
||||
"postgres-protocol",
|
||||
"postgres-types",
|
||||
"socket2 0.5.3",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-uring"
|
||||
version = "0.4.0"
|
||||
|
|
@ -2267,6 +2454,12 @@ version = "0.2.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
|
|
@ -2314,6 +2507,12 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urlencoding"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-cstr"
|
||||
version = "0.1.6"
|
||||
|
|
|
|||
|
|
@ -19,3 +19,4 @@ reqwest = { version = "0.11", features = ["json", "deflate", "gzip", "stream"] }
|
|||
eventsource-stream = "0.2"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
inotify = "0.10"
|
||||
tokio-postgres = "0.7"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
use std::sync::Arc;
|
||||
use tokio_postgres::{Client, Error, NoTls, Statement};
|
||||
|
||||
|
||||
const CREATE_SCHEMA_COMMANDS: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS instance_apps (host TEXT NOT NULL, client_id TEXT NOT NULL, client_secret TEXT NOT NULL, UNIQUE (host, client_id))",
|
||||
"CREATE TABLE IF NOT EXISTS instance_tokens (host TEXT NOT NULL, client_id TEXT NOT NULL, token TEXT NOT NULL, created TIMESTAMP)",
|
||||
"CREATE INDEX IF NOT EXISTS instance_tokens_host ON instance_tokens (host, created)",
|
||||
];
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Database {
|
||||
inner: Arc<DatabaseInner>,
|
||||
}
|
||||
|
||||
struct DatabaseInner {
|
||||
client: Client,
|
||||
add_app: Statement,
|
||||
get_apps: Statement,
|
||||
delete_app: Statement,
|
||||
add_token: Statement,
|
||||
get_token: Statement,
|
||||
delete_token: Statement,
|
||||
}
|
||||
|
||||
impl Database {
|
||||
pub async fn connect(conn_str: &str) -> Self {
|
||||
let (client, connection) = tokio_postgres::connect(conn_str, NoTls)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
tracing::error!("postgresql: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
let add_app = client.prepare("INSERT INTO instance_apps (host, client_id, client_secret) VALUES ($1, $2, $3)")
|
||||
.await
|
||||
.unwrap();
|
||||
let get_apps = client.prepare("SELECT client_id, client_secret FROM instance_apps WHERE host=$1 LIMIT 32")
|
||||
.await
|
||||
.unwrap();
|
||||
let delete_app = client.prepare("DELETE FROM instance_apps WHERE host=$1 AND client_id=$2")
|
||||
.await
|
||||
.unwrap();
|
||||
let add_token = client.prepare("INSERT INTO instance_tokens (host, client_id, token, created) VALUES ($1, $2, $3, NOW())")
|
||||
.await
|
||||
.unwrap();
|
||||
let get_token = client.prepare("SELECT token FROM instance_tokens WHERE host=$1 ORDER BY created ASC LIMIT 1")
|
||||
.await
|
||||
.unwrap();
|
||||
let delete_token = client.prepare("DELETE FROM instance_tokens WHERE host=$1 AND token=$2")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
Database {
|
||||
inner: Arc::new(DatabaseInner {
|
||||
client,
|
||||
add_app,
|
||||
get_apps,
|
||||
delete_app,
|
||||
add_token,
|
||||
get_token,
|
||||
delete_token,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_schema(&self) -> Result<(), Error> {
|
||||
for command in CREATE_SCHEMA_COMMANDS {
|
||||
self.inner.client.execute(*command, &[])
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn add_app(&self, host: &str, client_id: &str, client_secret: &str) -> Result<(), Error> {
|
||||
self.inner.client.execute(&self.inner.add_app, &[&host, &client_id, &client_secret])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_apps(&self, host: &str) -> Result<Vec<(String, String)>, Error> {
|
||||
let rows = self.inner.client.query(&self.inner.get_apps, &[&host])
|
||||
.await?;
|
||||
Ok(rows.into_iter().filter_map(|row| {
|
||||
let client_id = row.try_get(0).ok()?;
|
||||
let client_token = row.try_get(1).ok()?;
|
||||
Some((client_id, client_token))
|
||||
}).collect())
|
||||
}
|
||||
|
||||
pub async fn delete_app(&self, host: &str, client_id: &str) -> Result<(), Error> {
|
||||
self.inner.client.execute(&self.inner.delete_app, &[&host, &client_id])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn add_token(&self, host: &str, client_id: &str, token: &str) -> Result<(), Error> {
|
||||
self.inner.client.execute(&self.inner.add_token, &[&host, &client_id, &token])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_token(&self, host: &str) -> Result<Option<String>, Error> {
|
||||
let rows = self.inner.client.query(&self.inner.get_token, &[&host])
|
||||
.await?;
|
||||
Ok(rows.first().and_then(|row| row.try_get(0).ok()))
|
||||
}
|
||||
|
||||
pub async fn delete_token(&self, host: &str, token: &str) -> Result<(), Error> {
|
||||
self.inner.client.execute(&self.inner.delete_token, &[&host, &token])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ use std::{collections::{HashMap, HashSet}, time::Duration, ops::Deref};
|
|||
use chrono::{DateTime, FixedOffset};
|
||||
use futures::{Stream, StreamExt};
|
||||
use eventsource_stream::Eventsource;
|
||||
use reqwest::StatusCode;
|
||||
|
||||
pub fn url_host(url: &str) -> Option<String> {
|
||||
reqwest::Url::parse(url)
|
||||
|
|
@ -237,19 +238,19 @@ impl Feed {
|
|||
Ok(Feed { posts })
|
||||
}
|
||||
|
||||
pub async fn stream(client: &reqwest::Client, url: &str) -> Result<impl Stream<Item = EncodablePost>, String> {
|
||||
pub async fn stream(client: &reqwest::Client, url: &str) -> Result<impl Stream<Item = EncodablePost>, StreamError> {
|
||||
let res = client.get(url)
|
||||
.timeout(Duration::MAX)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("{}", e))?;
|
||||
.map_err(StreamError::Http)?;
|
||||
if res.status() != 200 {
|
||||
return Err(format!("HTTP {}", res.status()));
|
||||
return Err(StreamError::HttpStatus(res.status()));
|
||||
}
|
||||
let ct = res.headers().get("content-type")
|
||||
.and_then(|c| c.to_str().ok());
|
||||
if ct.map_or(true, |ct| ct != "text/event-stream") {
|
||||
return Err(format!("Invalid Content-Type: {:?}", ct));
|
||||
return Err(StreamError::InvalidContentType(ct.unwrap_or("").to_owned()));
|
||||
}
|
||||
|
||||
let src = res.bytes_stream().eventsource()
|
||||
|
|
@ -267,3 +268,22 @@ impl Feed {
|
|||
Ok(src)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum StreamError {
|
||||
HttpStatus(StatusCode),
|
||||
Http(reqwest::Error),
|
||||
InvalidContentType(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for StreamError {
|
||||
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
|
||||
match self {
|
||||
StreamError::HttpStatus(code) =>
|
||||
write!(fmt, "HTTP/{}", code),
|
||||
StreamError::Http(e) =>
|
||||
e.fmt(fmt),
|
||||
StreamError::InvalidContentType(ct) =>
|
||||
write!(fmt, "Invalid Content-Type: {}", ct),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ pub mod trend_tag;
|
|||
pub mod firehose;
|
||||
pub mod live_file;
|
||||
pub mod word_list;
|
||||
pub mod db;
|
||||
|
||||
pub const PERIODS: &[u64] = &[4, 24, 7 * 24];
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,5 @@ askama = "0.11"
|
|||
metrics = "0.20"
|
||||
metrics-util = "0.14"
|
||||
metrics-exporter-prometheus = "0.11"
|
||||
reqwest = "0.11"
|
||||
urlencoding = "1"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
redis: redis://10.233.12.2:6379/
|
||||
|
||||
database: host=127.0.0.1 port=5433 dbname=caveman username=caveman-gatherer password=c
|
||||
listen_port: 8000
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct Config {
|
||||
pub redis: String,
|
||||
pub database: String,
|
||||
pub listen_port: u16,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ use futures::future::{join, join_all};
|
|||
use cave::{
|
||||
firehose::FirehoseFactory,
|
||||
store::{Store, TREND_POOL_SIZE}, PERIODS,
|
||||
systemd,
|
||||
systemd, db::Database,
|
||||
};
|
||||
use http_body::combinators::UnsyncBoxBody;
|
||||
use metrics_exporter_prometheus::PrometheusHandle;
|
||||
|
|
@ -30,12 +30,15 @@ use crate::{
|
|||
trends::{TrendAnalyzer, TrendsResults},
|
||||
};
|
||||
|
||||
mod token_donate;
|
||||
mod token_collect;
|
||||
|
||||
type Languages = Vec<String>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ServerState {
|
||||
store: Store,
|
||||
db: Database,
|
||||
}
|
||||
|
||||
impl ServerState {
|
||||
|
|
@ -185,7 +188,6 @@ async fn streaming_api(
|
|||
.expect("Response")
|
||||
}
|
||||
|
||||
|
||||
async fn print_request(
|
||||
req: Request<Body>,
|
||||
next: Next<Body>,
|
||||
|
|
@ -206,6 +208,7 @@ async fn print_request(
|
|||
pub async fn start(
|
||||
listen_port: u16,
|
||||
store: Store,
|
||||
db: Database,
|
||||
firehose_factory: FirehoseFactory,
|
||||
recorder: PrometheusHandle,
|
||||
) {
|
||||
|
|
@ -216,7 +219,10 @@ pub async fn start(
|
|||
.route("/", get(home))
|
||||
.route("/in/:language", get(in_language))
|
||||
.route("/api/v1/streaming/public", get(streaming_api))
|
||||
.layer(Extension(ServerState { store }))
|
||||
.route("/token/donate", get(token_donate::get_token_donate).post(token_donate::post_token_donate))
|
||||
.route("/token/collect/:host", get(token_collect::get_token_collect))
|
||||
.route("/token/thanks", get(token_collect::get_token_thanks))
|
||||
.layer(Extension(ServerState { store, db }))
|
||||
.layer(Extension(firehose_factory))
|
||||
.route("/metrics", get(|| async move {
|
||||
recorder.render().into_response()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
use axum::{
|
||||
Extension,
|
||||
extract,
|
||||
http::StatusCode,
|
||||
response::{Html, IntoResponse},
|
||||
};
|
||||
use cave::db::Database;
|
||||
use crate::{
|
||||
http_server::ServerState,
|
||||
oauth,
|
||||
};
|
||||
|
||||
async fn collect_token(db: Database, host: &str, code: String) -> Result<(), String> {
|
||||
// try a few registered apps until one works
|
||||
for (client_id, client_secret) in db.get_apps(&host).await
|
||||
.map_err(|e| format!("{}", e))?
|
||||
{
|
||||
let app = oauth::Application {
|
||||
client_id,
|
||||
client_secret,
|
||||
};
|
||||
match app.obtain_token(&host, code.clone()).await {
|
||||
Ok(token) => {
|
||||
db.add_token(&host, &app.client_id, &token).await
|
||||
.expect("db.add_token");
|
||||
// success, done!
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("obtain_token for {}: {}", host, e);
|
||||
// app seems blocked, remove
|
||||
let _ = db.delete_app(host, &app.client_id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(format!("No registered app found for instance {}", host))
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct OAuthCode {
|
||||
code: String,
|
||||
}
|
||||
|
||||
pub async fn get_token_collect(
|
||||
Extension(ServerState { db, .. }): Extension<ServerState>,
|
||||
extract::Path(host): extract::Path<String>,
|
||||
extract::Query(OAuthCode { code }): extract::Query<OAuthCode>,
|
||||
) -> impl IntoResponse {
|
||||
match collect_token(db, &host, code.clone()).await {
|
||||
Ok(()) =>
|
||||
(
|
||||
StatusCode::SEE_OTHER,
|
||||
[("location", "/token/thanks")]
|
||||
).into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!("{}", e);
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
[("content-type", "text/plain")],
|
||||
e
|
||||
).into_response();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_token_thanks() -> impl IntoResponse {
|
||||
Html(&include_bytes!("../../templates/token_thanks.html")[..])
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
use axum::{
|
||||
Extension,
|
||||
extract,
|
||||
http::StatusCode,
|
||||
response::{Html, IntoResponse},
|
||||
};
|
||||
use crate::{
|
||||
http_server::ServerState,
|
||||
oauth,
|
||||
};
|
||||
|
||||
pub async fn get_token_donate() -> impl IntoResponse {
|
||||
Html(&include_bytes!("../../templates/token_donate.html")[..])
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize, Debug)]
|
||||
//#[allow(dead_code)]
|
||||
pub struct TokenDonateForm {
|
||||
instance: String,
|
||||
}
|
||||
|
||||
pub enum PostTokenDonateResult {
|
||||
OauthRedirect(String),
|
||||
Error(String),
|
||||
}
|
||||
|
||||
impl IntoResponse for PostTokenDonateResult {
|
||||
fn into_response(self) -> axum::response::Response {
|
||||
match self {
|
||||
PostTokenDonateResult::OauthRedirect(url) => (
|
||||
StatusCode::SEE_OTHER,
|
||||
[("location", url)]
|
||||
).into_response(),
|
||||
|
||||
PostTokenDonateResult::Error(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
[("content-type", "text/plain")],
|
||||
e
|
||||
).into_response(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn post_token_donate(
|
||||
Extension(ServerState { db, .. }): Extension<ServerState>,
|
||||
extract::Form(form): extract::Form<TokenDonateForm>,
|
||||
) -> PostTokenDonateResult {
|
||||
let apps = db.get_apps(&form.instance).await
|
||||
.expect("db.get_apps");
|
||||
|
||||
let app;
|
||||
if let Some((client_id, client_secret)) = apps.into_iter().next() {
|
||||
// existing app
|
||||
app = oauth::Application {
|
||||
client_id,
|
||||
client_secret,
|
||||
};
|
||||
} else {
|
||||
// register a new app with this instance
|
||||
app = match oauth::Application::register(&form.instance).await {
|
||||
Ok(app) => app,
|
||||
Err(e) => {
|
||||
tracing::error!("Canont register OAuth app: {}", e);
|
||||
return PostTokenDonateResult::Error(format!("{}", e));
|
||||
}
|
||||
};
|
||||
db.add_app(&form.instance, &app.client_id, &app.client_secret).await
|
||||
.expect("db.add_app");
|
||||
tracing::info!("Added app for {}: {}", form.instance, app.client_id);
|
||||
};
|
||||
|
||||
let url = app.generate_auth_url(&form.instance);
|
||||
PostTokenDonateResult::OauthRedirect(url)
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ mod config;
|
|||
mod trends;
|
||||
mod html_template;
|
||||
mod http_server;
|
||||
mod oauth;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
|
|
@ -25,6 +26,8 @@ async fn main() {
|
|||
.install_recorder()
|
||||
.unwrap();
|
||||
|
||||
cave::systemd::status("Connecting to database");
|
||||
let db = cave::db::Database::connect(&config.database).await;
|
||||
cave::systemd::status("Starting redis client");
|
||||
let store = cave::store::Store::new(8, config.redis.clone()).await;
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ async fn main() {
|
|||
let http = http_server::start(
|
||||
config.listen_port,
|
||||
store,
|
||||
db,
|
||||
firehose_factory,
|
||||
recorder,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
#[derive(serde::Serialize)]
|
||||
struct AppRegistration {
|
||||
client_name: String,
|
||||
redirect_uris: String,
|
||||
scopes: String,
|
||||
website: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct TokenRequest {
|
||||
grant_type: String,
|
||||
scope: String,
|
||||
client_id: String,
|
||||
client_secret: String,
|
||||
redirect_uri: String,
|
||||
code: String,
|
||||
}
|
||||
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct TokenResult {
|
||||
pub access_token: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct Application {
|
||||
pub client_id: String,
|
||||
pub client_secret: String,
|
||||
}
|
||||
|
||||
const SCOPES: &str = "read:statuses";
|
||||
|
||||
impl Application {
|
||||
pub async fn register(host: &str) -> Result<Self, reqwest::Error> {
|
||||
let url = format!("https://{}/api/v1/apps", host);
|
||||
let form = AppRegistration {
|
||||
client_name: "#FediBuzz".to_string(),
|
||||
website: "https://fedi.buzz/".to_string(),
|
||||
redirect_uris: Self::generate_redirect_url(host),
|
||||
scopes: SCOPES.to_string(),
|
||||
};
|
||||
let client = reqwest::Client::new();
|
||||
let res = client.post(url)
|
||||
.form(&form)
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub fn generate_redirect_url(host: &str) -> String {
|
||||
format!("https://fedi.buzz/token/collect/{}", host)
|
||||
}
|
||||
|
||||
pub fn generate_auth_url(&self, host: &str) -> String {
|
||||
format!(
|
||||
"https://{}/oauth/authorize?client_id={}&scope={}&response_type=code&redirect_uri={}",
|
||||
host,
|
||||
urlencoding::encode(&self.client_id),
|
||||
urlencoding::encode(SCOPES),
|
||||
urlencoding::encode(&Self::generate_redirect_url(host)),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn obtain_token(&self, host: &str, code: String) -> Result<String, reqwest::Error> {
|
||||
let url = format!("https://{}/oauth/token", host);
|
||||
let form = TokenRequest {
|
||||
grant_type: "authorization_code".to_string(),
|
||||
scope: SCOPES.to_string(),
|
||||
client_id: self.client_id.clone(),
|
||||
client_secret: self.client_secret.clone(),
|
||||
redirect_uri: Self::generate_redirect_url(host),
|
||||
code,
|
||||
};
|
||||
let client = reqwest::Client::new();
|
||||
let res: TokenResult = client.post(url)
|
||||
.form(&form)
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
Ok(res.access_token)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>#FediBuzz: Donate a token</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon16.png"/>
|
||||
<link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon48.png"/>
|
||||
<link rel="stylesheet" title="Default" type="text/css" href="/assets/style.css"/>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Fedi­Buzz</h1>
|
||||
<p>Donate an API token</p>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div>
|
||||
<form method="post">
|
||||
<fieldset>
|
||||
<legend>What is your home instance?</legend>
|
||||
<input name="instance" size="40">
|
||||
<input type="submit" value="Ok">
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
<h2>Reason</h2>
|
||||
<p>
|
||||
A token lets us access <a href="https://docs.joinmastodon.org/methods/streaming/#public">the public timeline streaming API</a> of your instance so that we can include it in our sampling of the Fediverse.
|
||||
</p>
|
||||
|
||||
<h2>Background</h2>
|
||||
<p>
|
||||
Mastodon is <a href="https://github.com/mastodon/mastodon/pull/23989#issuecomment-1628961709">breaking unauthenticated access</a> in future versions.
|
||||
</p>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>
|
||||
run by <a rel="me" href="https://c3d2.social/@astro">@astro­@c3d2.social</a>
|
||||
</p>
|
||||
<p>
|
||||
Get live ActivityPub content with the <a rel="me" href="https://relay.fedi.buzz">#FediBuzz relay</a>
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>#FediBuzz: Donate a token</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon16.png"/>
|
||||
<link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon48.png"/>
|
||||
<link rel="stylesheet" title="Default" type="text/css" href="/assets/style.css"/>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Thank you!</h1>
|
||||
<p>You donated an API token.</p>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div>
|
||||
<form method="post">
|
||||
<fieldset>
|
||||
<legend>Donate a token for another instance?</legend>
|
||||
<input name="instance" size="40">
|
||||
<input type="submit" value="Ok">
|
||||
</fieldset>
|
||||
</form>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>
|
||||
run by <a rel="me" href="https://c3d2.social/@astro">@astro­@c3d2.social</a>
|
||||
</p>
|
||||
<p>
|
||||
Get live ActivityPub content with the <a rel="me" href="https://relay.fedi.buzz">#FediBuzz relay</a>
|
||||
</p>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -20,6 +20,10 @@
|
|||
<header>
|
||||
<h1>Fedi­Buzz</h1>
|
||||
<p>Trends in the Fedi­verse</p>
|
||||
<!--p class="plea">
|
||||
#FediBuzz needs your help!
|
||||
<a href="/token/donate">Donate a token...</a>
|
||||
</p-->
|
||||
</header>
|
||||
|
||||
<nav>
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ metrics = "0.20"
|
|||
metrics-util = "0.14"
|
||||
metrics-exporter-prometheus = "0.11"
|
||||
serde_json = "1"
|
||||
urlencoding = "1"
|
||||
|
||||
[target.'cfg(not(target_env = "msvc"))'.dependencies]
|
||||
jemallocator = "0.5"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
redis: redis://10.233.12.2:6379/
|
||||
database: host=127.0.0.1 port=5433 dbname=caveman username=caveman-gatherer password=c
|
||||
|
||||
hosts:
|
||||
- chaos.social
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct Config {
|
||||
pub redis: String,
|
||||
pub database: String,
|
||||
pub hosts: Vec<String>,
|
||||
pub max_workers: usize,
|
||||
pub prometheus_port: u16,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ async fn run() {
|
|||
.install()
|
||||
.unwrap();
|
||||
|
||||
let db = cave::db::Database::connect(&config.database).await;
|
||||
let mut store = cave::store::Store::new(16, config.redis).await;
|
||||
let posts_cache = posts_cache::PostsCache::new(65536);
|
||||
|
||||
|
|
@ -128,6 +129,7 @@ async fn run() {
|
|||
.spawn(worker::run(
|
||||
message_tx.clone(),
|
||||
store.clone(),
|
||||
db.clone(),
|
||||
posts_cache.clone(),
|
||||
client.clone(),
|
||||
host
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ use std::collections::HashSet;
|
|||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use cave::feed::Post;
|
||||
use cave::{
|
||||
feed::{Feed, EncodablePost},
|
||||
db::Database,
|
||||
feed::{Feed, EncodablePost, Post, StreamError},
|
||||
store::Store,
|
||||
};
|
||||
use futures::{StreamExt, future};
|
||||
use reqwest::StatusCode;
|
||||
use crate::posts_cache::PostsCache;
|
||||
use crate::scheduler::{Host, InstanceHost};
|
||||
use crate::webfinger;
|
||||
|
|
@ -72,6 +73,7 @@ pub enum Message {
|
|||
pub async fn run(
|
||||
message_tx: tokio::sync::mpsc::UnboundedSender<Message>,
|
||||
store: Store,
|
||||
db: Database,
|
||||
posts_cache: PostsCache,
|
||||
client: reqwest::Client,
|
||||
host: InstanceHost,
|
||||
|
|
@ -87,7 +89,7 @@ pub async fn run(
|
|||
&client, robots_txt.clone(), &host.host
|
||||
),
|
||||
open_stream(
|
||||
message_tx.clone(), store.clone(),
|
||||
message_tx.clone(), store.clone(), db.clone(),
|
||||
&posts_cache,
|
||||
&client, robots_txt, host.host.clone()
|
||||
),
|
||||
|
|
@ -274,6 +276,7 @@ async fn process_posts(
|
|||
async fn open_stream(
|
||||
message_tx: tokio::sync::mpsc::UnboundedSender<Message>,
|
||||
store: Store,
|
||||
db: Database,
|
||||
posts_cache: &PostsCache,
|
||||
client: &reqwest::Client,
|
||||
robots_txt: RobotsTxt,
|
||||
|
|
@ -288,9 +291,32 @@ async fn open_stream(
|
|||
let posts_cache = posts_cache.clone();
|
||||
|
||||
metrics::increment_gauge!("hunter_requests", 1.0, "type" => "stream_open");
|
||||
let stream = Feed::stream(client, &url).await;
|
||||
let mut stream = Feed::stream(client, &url).await;
|
||||
metrics::decrement_gauge!("hunter_requests", 1.0, "type" => "stream_open");
|
||||
|
||||
let mut prev_token: Option<String> = None;
|
||||
while let Err(StreamError::HttpStatus(StatusCode::UNAUTHORIZED)) = &stream {
|
||||
if let Some(invalid_token) = prev_token {
|
||||
// If we tried with a token before but it's Unauthorized, delete it.
|
||||
let _ = db.delete_token(&host, &invalid_token).await;
|
||||
}
|
||||
|
||||
let token = db.get_token(&host).await
|
||||
.expect("db.get_token()");
|
||||
if let Some(token) = &token {
|
||||
let url = format!("https://{}/api/v1/streaming/public?access_token={}", host, urlencoding::encode(token));
|
||||
stream = Feed::stream(client, &url).await;
|
||||
} else {
|
||||
tracing::info!("No working token for {}", host);
|
||||
break;
|
||||
}
|
||||
prev_token = token;
|
||||
}
|
||||
|
||||
if let Err(e) = &stream {
|
||||
tracing::error!("Error opening stream to {}: {}", host, e);
|
||||
}
|
||||
|
||||
let stream = stream.map_err(|e| {
|
||||
format!("Stream error for {}: {}", host, e)
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ let
|
|||
|
||||
hunterDefaultSettings = {
|
||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||
hosts = [ "mastodon.social" "fosstodon.org" "chaos.social" "dresden.network" ];
|
||||
database = "host=/var/run/postgresql user=caveman-hunter dbname=caveman";
|
||||
hosts = [ "mastodon.social" ];
|
||||
max_workers = 16;
|
||||
prometheus_port = 9101;
|
||||
blocklist = blocklistPath;
|
||||
|
|
@ -33,6 +34,7 @@ let
|
|||
|
||||
gathererDefaultSettings = {
|
||||
redis = "redis://127.0.0.1:${toString cfg.redis.port}/";
|
||||
database = "host=/var/run/postgresql user=caveman-gatherer dbname=caveman";
|
||||
listen_port = 8000;
|
||||
};
|
||||
|
||||
|
|
@ -144,6 +146,16 @@ in
|
|||
maxmemory-policy = "allkeys-lru";
|
||||
};
|
||||
};
|
||||
services.postgresql = {
|
||||
enable = true;
|
||||
ensureDatabases = [ "caveman" ];
|
||||
ensureUsers = [ {
|
||||
name = databaseUser;
|
||||
ensurePermissions = {
|
||||
"DATABASE caveman" = "ALL PRIVILEGES";
|
||||
};
|
||||
} ];
|
||||
};
|
||||
|
||||
systemd.services.caveman-hunter = lib.mkIf cfg.hunter.enable {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
|
|||
Loading…
Reference in New Issue