refactor worker into worker/worker_job

This commit is contained in:
Astro 2019-10-11 22:15:38 +02:00
parent 54dc6db900
commit 844221616e
1 changed files with 125 additions and 116 deletions

View File

@ -73,20 +73,17 @@ impl Resources {
queue.pop_front() queue.pop_front()
} }
fn worker(&self) -> Result<(), Error> { fn worker_job(&self, db: &PgConnection) -> Result<bool, Error> {
let db = PgConnection::establish(&self.db_url)?;
loop {
let (id, cal_opts) = match self.next() { let (id, cal_opts) = match self.next() {
Some(next) => next, Some(next) => next,
None => return Ok(()), None => return Ok(true),
}; };
let (etag, last_modified) = db.transaction::<_, Error, _>(|| { let (etag, last_modified) = db.transaction::<_, Error, _>(|| {
let cals = calendars let cals = calendars
.filter(schema::calendars::dsl::id.eq(id.clone())) .filter(schema::calendars::dsl::id.eq(id.clone()))
.limit(1) .limit(1)
.load::<Calendar>(&db)?; .load::<Calendar>(db)?;
match cals.first() { match cals.first() {
None => { None => {
let cal = NewCalendar { let cal = NewCalendar {
@ -96,14 +93,14 @@ impl Resources {
}; };
diesel::insert_into(calendars) diesel::insert_into(calendars)
.values(&cal) .values(&cal)
.execute(&db)?; .execute(db)?;
Ok((None, None)) Ok((None, None))
} }
Some(cal) => { Some(cal) => {
diesel::update(calendars) diesel::update(calendars)
.filter(schema::calendars::dsl::id.eq(id.clone())) .filter(schema::calendars::dsl::id.eq(id.clone()))
.set(schema::calendars::dsl::last_fetch.eq(Utc::now().naive_utc())) .set(schema::calendars::dsl::last_fetch.eq(Utc::now().naive_utc()))
.execute(&db)?; .execute(db)?;
let result = let result =
// Use ETag/Last-Modified only if URL hasn't changed // Use ETag/Last-Modified only if URL hasn't changed
if cal.url == cal_opts.url { if cal.url == cal_opts.url {
@ -135,8 +132,8 @@ impl Resources {
.filter(schema::calendars::dsl::id.eq(id.clone())) .filter(schema::calendars::dsl::id.eq(id.clone()))
.set((schema::calendars::dsl::last_success.eq(Utc::now().naive_utc()), .set((schema::calendars::dsl::last_success.eq(Utc::now().naive_utc()),
schema::calendars::dsl::error_message.eq(msg))) schema::calendars::dsl::error_message.eq(msg)))
.execute(&db)?; .execute(db)?;
return Ok(()) return Ok(false)
} }
let etag = res.headers().get(ETAG) let etag = res.headers().get(ETAG)
@ -152,7 +149,7 @@ impl Resources {
.set((schema::calendars::dsl::last_success.eq(Utc::now().naive_utc()), .set((schema::calendars::dsl::last_success.eq(Utc::now().naive_utc()),
schema::calendars::dsl::etag.eq(etag), schema::calendars::dsl::etag.eq(etag),
schema::calendars::dsl::last_modified.eq(last_modified))) schema::calendars::dsl::last_modified.eq(last_modified)))
.execute(&db)?; .execute(db)?;
let mut p = Parser::new(); let mut p = Parser::new();
let mut buf = [0; 1024]; let mut buf = [0; 1024];
@ -184,16 +181,28 @@ impl Resources {
println!("[{}] {} events", id, events.len()); println!("[{}] {} events", id, events.len());
diesel::delete(schema::events::dsl::events) diesel::delete(schema::events::dsl::events)
.filter(schema::events::dsl::calendar.eq(id)) .filter(schema::events::dsl::calendar.eq(id))
.execute(&db)?; .execute(db)?;
for event in events.values() { for event in events.values() {
diesel::insert_into(schema::events::dsl::events) diesel::insert_into(schema::events::dsl::events)
.values(event) .values(event)
.execute(&db)?; .execute(db)?;
} }
Ok(()) Ok(false)
})?; })?;
Ok(false)
} }
fn worker_loop(&self) -> Result<(), Error> {
let db = PgConnection::establish(&self.db_url)?;
let mut done = false;
while ! done {
done = self.worker_job(&db)
.map_err(|e| println!("{:?}", e))
.unwrap_or(false);
}
Ok(())
} }
} }
@ -244,7 +253,7 @@ fn main() {
let cpus = num_cpus::get(); let cpus = num_cpus::get();
let workers: Vec<_> = (0..cpus) let workers: Vec<_> = (0..cpus)
.map(|_| s.spawn(|_| { .map(|_| s.spawn(|_| {
res.worker() res.worker_loop()
.map_err(|e| println!("{:?}", e)) .map_err(|e| println!("{:?}", e))
})) }))
.collect(); .collect();