You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

26 lines
819 B

from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class QuotesSpider(scrapy.Spider):
name = "quotes"
def start_requests(self):
urls = [
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for author in response.xpath('//div[@class="quote"]/span/small[@class="author"]/text()'):
self.log('Author %s' % author.get())
runner = CrawlerRunner()
d = runner.crawl(QuotesSpider)
d.addBoth(lambda _: reactor.stop()) # the script will block here until the crawling is finished