use env_logger
This commit is contained in:
parent
293e26e0ec
commit
748ccd4f59
58
hunter/Cargo.lock
generated
58
hunter/Cargo.lock
generated
|
@ -8,6 +8,15 @@ version = "1.0.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "0.7.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "android_system_properties"
|
name = "android_system_properties"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
|
@ -47,6 +56,17 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "atty"
|
||||||
|
version = "0.2.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||||
|
dependencies = [
|
||||||
|
"hermit-abi",
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "autocfg"
|
name = "autocfg"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
@ -82,6 +102,8 @@ name = "caveman"
|
||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"env_logger",
|
||||||
|
"log",
|
||||||
"redis",
|
"redis",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
|
@ -218,6 +240,19 @@ dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "env_logger"
|
||||||
|
version = "0.9.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272"
|
||||||
|
dependencies = [
|
||||||
|
"atty",
|
||||||
|
"humantime",
|
||||||
|
"log",
|
||||||
|
"regex",
|
||||||
|
"termcolor",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastrand"
|
name = "fastrand"
|
||||||
version = "1.8.0"
|
version = "1.8.0"
|
||||||
|
@ -424,6 +459,12 @@ version = "1.0.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "humantime"
|
||||||
|
version = "2.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper"
|
name = "hyper"
|
||||||
version = "0.14.22"
|
version = "0.14.22"
|
||||||
|
@ -802,6 +843,23 @@ dependencies = [
|
||||||
"bitflags",
|
"bitflags",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.6.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "remove_dir_all"
|
name = "remove_dir_all"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
|
|
|
@ -10,3 +10,5 @@ serde = { version = "1", features = ["derive"] }
|
||||||
serde_yaml = "0.9"
|
serde_yaml = "0.9"
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
redis = { version = "0.22", features = ["tokio-comp", "connection-manager"] }
|
redis = { version = "0.22", features = ["tokio-comp", "connection-manager"] }
|
||||||
|
log = "0.4"
|
||||||
|
env_logger = "0.9"
|
||||||
|
|
|
@ -88,7 +88,7 @@ impl Feed {
|
||||||
.await?
|
.await?
|
||||||
.json()
|
.json()
|
||||||
.await?;
|
.await?;
|
||||||
println!("{} {} posts", url, posts.len());
|
log::debug!("{} {} posts", url, posts.len());
|
||||||
Ok(Feed { posts })
|
Ok(Feed { posts })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,8 @@ async fn main() {
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
env_logger::init();
|
||||||
|
|
||||||
let config = config::Config::load_file(
|
let config = config::Config::load_file(
|
||||||
&std::env::args()
|
&std::env::args()
|
||||||
.skip(1)
|
.skip(1)
|
||||||
|
@ -51,7 +53,7 @@ async fn main() {
|
||||||
let mut workers_active = 0usize;
|
let mut workers_active = 0usize;
|
||||||
let (message_tx, mut message_rx) = tokio::sync::mpsc::unbounded_channel();
|
let (message_tx, mut message_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||||
loop {
|
loop {
|
||||||
println!("{} workers active, queued {} of {}", workers_active, scheduler.queue_len(), scheduler.size());
|
log::trace!("{} workers active, queued {} of {}", workers_active, scheduler.queue_len(), scheduler.size());
|
||||||
let next_task = if workers_active < config.max_workers {
|
let next_task = if workers_active < config.max_workers {
|
||||||
scheduler.dequeue()
|
scheduler.dequeue()
|
||||||
} else {
|
} else {
|
||||||
|
@ -79,7 +81,6 @@ async fn main() {
|
||||||
}).await;
|
}).await;
|
||||||
}
|
}
|
||||||
Ok(host) => {
|
Ok(host) => {
|
||||||
println!("Fetch {}", host);
|
|
||||||
workers_active += 1;
|
workers_active += 1;
|
||||||
worker::fetch_and_process(
|
worker::fetch_and_process(
|
||||||
message_tx.clone(),
|
message_tx.clone(),
|
||||||
|
|
|
@ -77,7 +77,7 @@ async fn save_post_tags(man: &mut redis::aio::ConnectionManager, post: Post) {
|
||||||
match cmd.query_async(man).await {
|
match cmd.query_async(man).await {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("redis error: {:?}", e);
|
log::error!("redis error: {:?}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -75,9 +75,9 @@ impl Scheduler {
|
||||||
.unwrap_or(Err(Duration::from_secs(1)))
|
.unwrap_or(Err(Duration::from_secs(1)))
|
||||||
.map(|host| {
|
.map(|host| {
|
||||||
if let Some(last_fetch) = self.instances.get(&host).and_then(|i| i.last_fetch) {
|
if let Some(last_fetch) = self.instances.get(&host).and_then(|i| i.last_fetch) {
|
||||||
println!("Fetch {} - last before {:?}", host, now - last_fetch);
|
log::debug!("Fetch {} - last before {:?}", host, now - last_fetch);
|
||||||
} else {
|
} else {
|
||||||
println!("Fetch {} - NEW", host);
|
log::debug!("Fetch {} - NEW", host);
|
||||||
}
|
}
|
||||||
host
|
host
|
||||||
})
|
})
|
||||||
|
@ -85,7 +85,7 @@ impl Scheduler {
|
||||||
Err(time - now)
|
Err(time - now)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
println!("empty queue");
|
log::warn!("empty queue");
|
||||||
Err(Duration::from_secs(60))
|
Err(Duration::from_secs(60))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -75,7 +75,7 @@ pub fn fetch_and_process(
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
println!("Failed fetching {}: {}", host, e);
|
log::error!("Failed fetching {}: {}", host, e);
|
||||||
message_tx.send(Message::Error { host }).unwrap();
|
message_tx.send(Message::Error { host }).unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user