From 2a5ef5ffdf6079fe9d615127c507d74dcc935e64 Mon Sep 17 00:00:00 2001 From: indigoclay Date: Fri, 9 Aug 2019 08:16:46 +0200 Subject: [PATCH] added scrapy example --- .gitignore | 2 + example_scr/scrapy/tutorial/items.json | 0 example_scr/scrapy/tutorial/quotes-1.html | 281 ++++++++++++++++ example_scr/scrapy/tutorial/quotes-2.html | 301 ++++++++++++++++++ example_scr/scrapy/tutorial/scrapy.cfg | 11 + example_scr/scrapy/tutorial/tutorial.pyproj | 40 +++ .../scrapy/tutorial/tutorial/__init__.py | 0 example_scr/scrapy/tutorial/tutorial/items.py | 14 + .../scrapy/tutorial/tutorial/middlewares.py | 103 ++++++ .../scrapy/tutorial/tutorial/pipelines.py | 11 + .../scrapy/tutorial/tutorial/settings.py | 90 ++++++ .../tutorial/tutorial/spiders/__init__.py | 4 + .../tutorial/spiders/quotes_spider.py | 26 ++ 13 files changed, 883 insertions(+) create mode 100644 example_scr/scrapy/tutorial/items.json create mode 100644 example_scr/scrapy/tutorial/quotes-1.html create mode 100644 example_scr/scrapy/tutorial/quotes-2.html create mode 100644 example_scr/scrapy/tutorial/scrapy.cfg create mode 100644 example_scr/scrapy/tutorial/tutorial.pyproj create mode 100644 example_scr/scrapy/tutorial/tutorial/__init__.py create mode 100644 example_scr/scrapy/tutorial/tutorial/items.py create mode 100644 example_scr/scrapy/tutorial/tutorial/middlewares.py create mode 100644 example_scr/scrapy/tutorial/tutorial/pipelines.py create mode 100644 example_scr/scrapy/tutorial/tutorial/settings.py create mode 100644 example_scr/scrapy/tutorial/tutorial/spiders/__init__.py create mode 100644 example_scr/scrapy/tutorial/tutorial/spiders/quotes_spider.py diff --git a/.gitignore b/.gitignore index e5a150b..bd92e82 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ data/* result/* docs/* *.pyc +.vs/* +/example_scr/scrapy/tutorial/.vs/* \ No newline at end of file diff --git a/example_scr/scrapy/tutorial/items.json b/example_scr/scrapy/tutorial/items.json new file mode 100644 index 0000000..e69de29 diff --git a/example_scr/scrapy/tutorial/quotes-1.html b/example_scr/scrapy/tutorial/quotes-1.html new file mode 100644 index 0000000..71aff88 --- /dev/null +++ b/example_scr/scrapy/tutorial/quotes-1.html @@ -0,0 +1,281 @@ + + + + + Quotes to Scrape + + + + +
+
+
+

+ Quotes to Scrape +

+
+
+

+ + Login + +

+
+
+ + +
+
+ +
+ “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.” + by + (about) + +
+ Tags: + + + change + + deep-thoughts + + thinking + + world + +
+
+ +
+ “It is our choices, Harry, that show what we truly are, far more than our abilities.” + by + (about) + +
+ Tags: + + + abilities + + choices + +
+
+ +
+ “There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.” + by + (about) + +
+ Tags: + + + inspirational + + life + + live + + miracle + + miracles + +
+
+ +
+ “The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.” + by + (about) + +
+ Tags: + + + aliteracy + + books + + classic + + humor + +
+
+ +
+ “Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.” + by + (about) + +
+ Tags: + + + be-yourself + + inspirational + +
+
+ +
+ “Try not to become a man of success. Rather become a man of value.” + by + (about) + +
+ Tags: + + + adulthood + + success + + value + +
+
+ +
+ “It is better to be hated for what you are than to be loved for what you are not.” + by + (about) + +
+ Tags: + + + life + + love + +
+
+ +
+ “I have not failed. I've just found 10,000 ways that won't work.” + by + (about) + +
+ Tags: + + + edison + + failure + + inspirational + + paraphrased + +
+
+ +
+ “A woman is like a tea bag; you never know how strong it is until it's in hot water.” + by + (about) + +
+ Tags: + + + misattributed-eleanor-roosevelt + +
+
+ +
+ “A day without sunshine is like, you know, night.” + by + (about) + +
+ Tags: + + + humor + + obvious + + simile + +
+
+ + +
+
+ +

Top Ten tags

+ + + love + + + + inspirational + + + + life + + + + humor + + + + books + + + + reading + + + + friendship + + + + friends + + + + truth + + + + simile + + + +
+
+ +
+ + + \ No newline at end of file diff --git a/example_scr/scrapy/tutorial/quotes-2.html b/example_scr/scrapy/tutorial/quotes-2.html new file mode 100644 index 0000000..3e04f31 --- /dev/null +++ b/example_scr/scrapy/tutorial/quotes-2.html @@ -0,0 +1,301 @@ + + + + + Quotes to Scrape + + + + +
+
+
+

+ Quotes to Scrape +

+
+
+

+ + Login + +

+
+
+ + +
+
+ +
+ “This life is what you make it. No matter what, you're going to mess up sometimes, it's a universal truth. But the good part is you get to decide how you're going to mess it up. Girls will be your friends - they'll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they're your true best friends. Don't let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they'll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can't give up because if you give up, you'll never find your soulmate. You'll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn't mean you're gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don't, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life's a beautiful thing and there's so much to smile about.” + by + (about) + +
+ Tags: + + + friends + + heartbreak + + inspirational + + life + + love + + sisters + +
+
+ +
+ “It takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.” + by + (about) + +
+ Tags: + + + courage + + friends + +
+
+ +
+ “If you can't explain it to a six year old, you don't understand it yourself.” + by + (about) + +
+ Tags: + + + simplicity + + understand + +
+
+ +
+ “You may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She's not perfect—you aren't either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break—her heart. So don't hurt her, don't change her, don't analyze and don't expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she's not there.” + by + (about) + +
+ Tags: + + + love + +
+
+ +
+ “I like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.” + by + (about) + +
+ Tags: + + + fantasy + +
+
+ +
+ “I may not have gone where I intended to go, but I think I have ended up where I needed to be.” + by + (about) + +
+ Tags: + + + life + + navigation + +
+
+ +
+ “The opposite of love is not hate, it's indifference. The opposite of art is not ugliness, it's indifference. The opposite of faith is not heresy, it's indifference. And the opposite of life is not death, it's indifference.” + by + (about) + +
+ Tags: + + + activism + + apathy + + hate + + indifference + + inspirational + + love + + opposite + + philosophy + +
+
+ +
+ “It is not a lack of love, but a lack of friendship that makes unhappy marriages.” + by + (about) + +
+ Tags: + + + friendship + + lack-of-friendship + + lack-of-love + + love + + marriage + + unhappy-marriage + +
+
+ +
+ “Good friends, good books, and a sleepy conscience: this is the ideal life.” + by + (about) + +
+ Tags: + + + books + + contentment + + friends + + friendship + + life + +
+
+ +
+ “Life is what happens to us while we are making other plans.” + by + (about) + +
+ Tags: + + + fate + + life + + misattributed-john-lennon + + planning + + plans + +
+
+ + +
+
+ +

Top Ten tags

+ + + love + + + + inspirational + + + + life + + + + humor + + + + books + + + + reading + + + + friendship + + + + friends + + + + truth + + + + simile + + + +
+
+ +
+ + + \ No newline at end of file diff --git a/example_scr/scrapy/tutorial/scrapy.cfg b/example_scr/scrapy/tutorial/scrapy.cfg new file mode 100644 index 0000000..d79f6f1 --- /dev/null +++ b/example_scr/scrapy/tutorial/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = tutorial.settings + +[deploy] +#url = http://localhost:6800/ +project = tutorial diff --git a/example_scr/scrapy/tutorial/tutorial.pyproj b/example_scr/scrapy/tutorial/tutorial.pyproj new file mode 100644 index 0000000..20bfab6 --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial.pyproj @@ -0,0 +1,40 @@ + + + + Debug + 2.0 + {1d3e377b-fd78-4311-849e-104153e87810} + + tutorial\spiders\quotes_spider.py + + . + . + {888888a0-9f3d-457c-b088-3a5042f75d52} + Standard Python launcher + CondaEnv|CondaEnv|ngo + + + + + 10.0 + + + + + + + Code + + + + + + + + + + + + + + \ No newline at end of file diff --git a/example_scr/scrapy/tutorial/tutorial/__init__.py b/example_scr/scrapy/tutorial/tutorial/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/example_scr/scrapy/tutorial/tutorial/items.py b/example_scr/scrapy/tutorial/tutorial/items.py new file mode 100644 index 0000000..569bb16 --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/items.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class TutorialItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + pass diff --git a/example_scr/scrapy/tutorial/tutorial/middlewares.py b/example_scr/scrapy/tutorial/tutorial/middlewares.py new file mode 100644 index 0000000..79182cb --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/middlewares.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class TutorialSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class TutorialDownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/example_scr/scrapy/tutorial/tutorial/pipelines.py b/example_scr/scrapy/tutorial/tutorial/pipelines.py new file mode 100644 index 0000000..65c1937 --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/pipelines.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html + + +class TutorialPipeline(object): + def process_item(self, item, spider): + return item diff --git a/example_scr/scrapy/tutorial/tutorial/settings.py b/example_scr/scrapy/tutorial/tutorial/settings.py new file mode 100644 index 0000000..66cd185 --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/settings.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for tutorial project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'tutorial' + +SPIDER_MODULES = ['tutorial.spiders'] +NEWSPIDER_MODULE = 'tutorial.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'tutorial (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'tutorial.middlewares.TutorialSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'tutorial.middlewares.TutorialDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://doc.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# 'tutorial.pipelines.TutorialPipeline': 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/example_scr/scrapy/tutorial/tutorial/spiders/__init__.py b/example_scr/scrapy/tutorial/tutorial/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/example_scr/scrapy/tutorial/tutorial/spiders/quotes_spider.py b/example_scr/scrapy/tutorial/tutorial/spiders/quotes_spider.py new file mode 100644 index 0000000..8723d22 --- /dev/null +++ b/example_scr/scrapy/tutorial/tutorial/spiders/quotes_spider.py @@ -0,0 +1,26 @@ +from twisted.internet import reactor +import scrapy +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + +class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for author in response.xpath('//div[@class="quote"]/span/small[@class="author"]/text()'): + self.log('Author %s' % author.get()) + print(author.get()) + +runner = CrawlerRunner() + +d = runner.crawl(QuotesSpider) +d.addBoth(lambda _: reactor.stop()) +reactor.run() # the script will block here until the crawling is finished \ No newline at end of file