Compare commits

..

4 Commits

Author SHA1 Message Date
Indigo 8294371fe6 some text change 2019-08-08 15:24:39 +02:00
Indigo 91a1882ea9 added subbranch 2019-08-08 15:12:01 +02:00
Indigo ab6a069d9c added testtext 2019-08-08 15:05:22 +02:00
Indigo 80c827a7e2 added requirements and howto 2019-08-08 14:57:51 +02:00
26 changed files with 2 additions and 1199 deletions

4
.gitignore vendored
View File

@ -2,7 +2,3 @@ data/*
result/*
docs/*
*.pyc
.vs/*
/example_scr/scrapy/tutorial/.vs/*
*.sln
*.pyproj

View File

@ -1,7 +0,0 @@
wie asynchrone datenbanken zugriffe (schreiben) behandeln?
oder sind alle schreibzugriffe synchron?
bei datenbankabfragen: läuft python ansynchron im hintergrund?
datenbankstruktur:
wie werden eher statische daten wie "Organisation" behandelt? händisch eigetragen? geparst ? => wenn ja wie werden die daten extrahiert?
letztes gild auch zum beispiel für daten wie location? wie automatisieren? jede location manuell nachpflegen ?

View File

@ -1,281 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
<a class="tag" href="/tag/choices/page/1/">choices</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/live/page/1/">live</a>
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
<span>by <small class="author" itemprop="author">Jane Austen</small>
<a href="/author/Jane-Austen">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/classic/page/1/">classic</a>
<a class="tag" href="/tag/humor/page/1/">humor</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
<a class="tag" href="/tag/success/page/1/">success</a>
<a class="tag" href="/tag/value/page/1/">value</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
<span>by <small class="author" itemprop="author">André Gide</small>
<a href="/author/Andre-Gide">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,love" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
<a href="/author/Thomas-A-Edison">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
<a class="tag" href="/tag/edison/page/1/">edison</a>
<a class="tag" href="/tag/failure/page/1/">failure</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
<a href="/author/Eleanor-Roosevelt">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
<span>by <small class="author" itemprop="author">Steve Martin</small>
<a href="/author/Steve-Martin">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
<a class="tag" href="/tag/humor/page/1/">humor</a>
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
<a class="tag" href="/tag/simile/page/1/">simile</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="next">
<a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'></span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

View File

@ -1,301 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“This life is what you make it. No matter what, you&#39;re going to mess up sometimes, it&#39;s a universal truth. But the good part is you get to decide how you&#39;re going to mess it up. Girls will be your friends - they&#39;ll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they&#39;re your true best friends. Don&#39;t let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they&#39;ll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can&#39;t give up because if you give up, you&#39;ll never find your soulmate. You&#39;ll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn&#39;t mean you&#39;re gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don&#39;t, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life&#39;s a beautiful thing and there&#39;s so much to smile about.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="friends,heartbreak,inspirational,life,love,sisters" / >
<a class="tag" href="/tag/friends/page/1/">friends</a>
<a class="tag" href="/tag/heartbreak/page/1/">heartbreak</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
<a class="tag" href="/tag/sisters/page/1/">sisters</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="courage,friends" / >
<a class="tag" href="/tag/courage/page/1/">courage</a>
<a class="tag" href="/tag/friends/page/1/">friends</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“If you can&#39;t explain it to a six year old, you don&#39;t understand it yourself.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="simplicity,understand" / >
<a class="tag" href="/tag/simplicity/page/1/">simplicity</a>
<a class="tag" href="/tag/understand/page/1/">understand</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“You may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She&#39;s not perfect—you aren&#39;t either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break—her heart. So don&#39;t hurt her, don&#39;t change her, don&#39;t analyze and don&#39;t expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she&#39;s not there.”</span>
<span>by <small class="author" itemprop="author">Bob Marley</small>
<a href="/author/Bob-Marley">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="love" / >
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.”</span>
<span>by <small class="author" itemprop="author">Dr. Seuss</small>
<a href="/author/Dr-Seuss">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="fantasy" / >
<a class="tag" href="/tag/fantasy/page/1/">fantasy</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I may not have gone where I intended to go, but I think I have ended up where I needed to be.”</span>
<span>by <small class="author" itemprop="author">Douglas Adams</small>
<a href="/author/Douglas-Adams">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,navigation" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/navigation/page/1/">navigation</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The opposite of love is not hate, it&#39;s indifference. The opposite of art is not ugliness, it&#39;s indifference. The opposite of faith is not heresy, it&#39;s indifference. And the opposite of life is not death, it&#39;s indifference.”</span>
<span>by <small class="author" itemprop="author">Elie Wiesel</small>
<a href="/author/Elie-Wiesel">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="activism,apathy,hate,indifference,inspirational,love,opposite,philosophy" / >
<a class="tag" href="/tag/activism/page/1/">activism</a>
<a class="tag" href="/tag/apathy/page/1/">apathy</a>
<a class="tag" href="/tag/hate/page/1/">hate</a>
<a class="tag" href="/tag/indifference/page/1/">indifference</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/love/page/1/">love</a>
<a class="tag" href="/tag/opposite/page/1/">opposite</a>
<a class="tag" href="/tag/philosophy/page/1/">philosophy</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is not a lack of love, but a lack of friendship that makes unhappy marriages.”</span>
<span>by <small class="author" itemprop="author">Friedrich Nietzsche</small>
<a href="/author/Friedrich-Nietzsche">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="friendship,lack-of-friendship,lack-of-love,love,marriage,unhappy-marriage" / >
<a class="tag" href="/tag/friendship/page/1/">friendship</a>
<a class="tag" href="/tag/lack-of-friendship/page/1/">lack-of-friendship</a>
<a class="tag" href="/tag/lack-of-love/page/1/">lack-of-love</a>
<a class="tag" href="/tag/love/page/1/">love</a>
<a class="tag" href="/tag/marriage/page/1/">marriage</a>
<a class="tag" href="/tag/unhappy-marriage/page/1/">unhappy-marriage</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Good friends, good books, and a sleepy conscience: this is the ideal life.”</span>
<span>by <small class="author" itemprop="author">Mark Twain</small>
<a href="/author/Mark-Twain">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="books,contentment,friends,friendship,life" / >
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/contentment/page/1/">contentment</a>
<a class="tag" href="/tag/friends/page/1/">friends</a>
<a class="tag" href="/tag/friendship/page/1/">friendship</a>
<a class="tag" href="/tag/life/page/1/">life</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Life is what happens to us while we are making other plans.”</span>
<span>by <small class="author" itemprop="author">Allen Saunders</small>
<a href="/author/Allen-Saunders">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="fate,life,misattributed-john-lennon,planning,plans" / >
<a class="tag" href="/tag/fate/page/1/">fate</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/misattributed-john-lennon/page/1/">misattributed-john-lennon</a>
<a class="tag" href="/tag/planning/page/1/">planning</a>
<a class="tag" href="/tag/plans/page/1/">plans</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="previous">
<a href="/page/1/"><span aria-hidden="true">&larr;</span> Previous</a>
</li>
<li class="next">
<a href="/page/3/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'></span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

View File

@ -1,11 +0,0 @@
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = tutorial.settings
[deploy]
#url = http://localhost:6800/
project = tutorial

View File

@ -1,40 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{1d3e377b-fd78-4311-849e-104153e87810}</ProjectGuid>
<ProjectHome />
<StartupFile>tutorial\spiders\quotes_spider.py</StartupFile>
<SearchPath />
<WorkingDirectory>.</WorkingDirectory>
<OutputPath>.</OutputPath>
<ProjectTypeGuids>{888888a0-9f3d-457c-b088-3a5042f75d52}</ProjectTypeGuids>
<LaunchProvider>Standard Python launcher</LaunchProvider>
<InterpreterId>CondaEnv|CondaEnv|ngo</InterpreterId>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
<PropertyGroup Condition="'$(Configuration)' == 'Release'" />
<PropertyGroup>
<VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
</PropertyGroup>
<ItemGroup>
<Compile Include="tutorial\items.py" />
<Compile Include="tutorial\middlewares.py" />
<Compile Include="tutorial\pipelines.py" />
<Compile Include="tutorial\spiders\quotes_spider.py">
<SubType>Code</SubType>
</Compile>
<Compile Include="tutorial\settings.py" />
<Compile Include="tutorial\spiders\__init__.py" />
<Compile Include="tutorial\__init__.py" />
</ItemGroup>
<ItemGroup>
<Folder Include="tutorial" />
<Folder Include="tutorial\spiders" />
</ItemGroup>
<ItemGroup>
<InterpreterReference Include="CondaEnv|CondaEnv|ngo" />
</ItemGroup>
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
</Project>

View File

@ -1,14 +0,0 @@
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class TutorialItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass

View File

@ -1,103 +0,0 @@
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class TutorialSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesnt have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class TutorialDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)

View File

@ -1,11 +0,0 @@
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class TutorialPipeline(object):
def process_item(self, item, spider):
return item

View File

@ -1,90 +0,0 @@
# -*- coding: utf-8 -*-
# Scrapy settings for tutorial project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'tutorial'
SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'tutorial (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'tutorial.middlewares.TutorialSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'tutorial.middlewares.TutorialDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'tutorial.pipelines.TutorialPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

View File

@ -1,4 +0,0 @@
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.

View File

@ -1,26 +0,0 @@
from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class QuotesSpider(scrapy.Spider):
name = "quotes"
def start_requests(self):
urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for author in response.xpath('//div[@class="quote"]/span/small[@class="author"]/text()'):
self.log('Author %s' % author.get())
print(author.get())
runner = CrawlerRunner()
d = runner.crawl(QuotesSpider)
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished

View File

@ -1,2 +0,0 @@
use scrapy & sqlalchemy together
https://www.accordbox.com/blog/scrapy-tutorial-9-how-use-scrapy-item/

View File

@ -1,24 +0,0 @@
### another way (but again *not the only way*) to do it ###
from contextlib import contextmanager
@contextmanager
def session_scope():
"""Provide a transactional scope around a series of operations."""
session = Session()
try:
yield session
session.commit()
except:
session.rollback()
raise
finally:
session.close()
#all session.add will be done in session_scope(). tbd: remove individual commits from ORM_test.py
def run_my_program():
with session_scope() as session:
pass
#ThingOne().go(session)
#ThingTwo().go(session)

View File

@ -21,7 +21,7 @@ def main():
# exist.""")
# parser.add_argument('msgfile')
# args = parser.parse_args()
''' All of the above is commented out because not necessary in this case'''
with open("mysourcedirectory/mymailfilename", 'rb') as fp:

View File

@ -1,24 +0,0 @@
import sqlalchemy
from sqlalchemy import Index
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base
from src.ORM_modules.base import Base, NgoBase
class Event(Base, NgoBase):
#__tablename__ = 'events'
EID = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
ETitle = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=False)
Edate = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=False)
EUrl = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=True)
#organisation = relationship('Organisation', secondary='TOrganisation_Events')
def getIndex(self):
return self.EID
__table_args__ = (Index('ETitle', "ETitle", "Edate", "EUrl"), )
def __repr__(self):
return "<Event(name='{0}', fullname='{1}', nickname='{2}')>".format(
self.ETitle, self.Edate, self.EUrl)

View File

@ -1,72 +0,0 @@
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from src.ORM_modules.Event import Event
from src.ORM_modules.Organisation import Organisation
from src.ORM_modules.Relationships import Organisation_Events
from src.ORM_modules.base import Base
from sqlalchemy import and_
# Define the MySQL engine using MySQL Connector/Python
engine = sqlalchemy.create_engine('mysql+mysqlconnector://admin:admin@localhost/wordpress', echo=True)
Base.metadata.create_all(engine)
# Create a session
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
session = Session()
eventName = 'event2';
organisationName = 'Supger GMBH2';
# create these objects somwehre, check if their primary keys (names) exists
__event = Event(ETitle=eventName, Edate='20.07.2019', EUrl='www.hallo.de')
__organisation = Organisation(OName=organisationName, ODescription='Super Orga')
rcolumnAttribute = Event.ETitle
lcolumnAttribute = Organisation.OName
assocType = Organisation_Events
def commit_ltarget_rtarget(__linstance, __rinstance, rcolumnAttribute, lcolumnAttribute, assocType):
attrKey = rcolumnAttribute.__dict__['key']
rcolumnAttributeValue = getattr(__rinstance,attrKey)
rcolumnType = type(__rinstance)
_rinstance = session.query(rcolumnType).filter(rcolumnAttribute == rcolumnAttributeValue).first()
attrKey = lcolumnAttribute.__dict__['key']
lcolumnAttributeValue = getattr(__linstance,attrKey)
lcolumnType = type(__linstance)
#check if already in DB
_linstance = session.query(lcolumnType).filter(lcolumnAttribute == lcolumnAttributeValue).first()
_assoc = None
# many to many:
# if both exist dont create either, but check association. => it's possible that an event is there + organisations is there but the event is created by another organisation
if ((_rinstance != None) and (_linstance != None)):
lFK = assocType.getLeftFK()
rFK = assocType.getRightFK()
_assoc = session.query(assocType).filter(and_(rFK == _rinstance.getIndex(), lFK == _linstance.getIndex())).first()
if (_assoc == None):
# create only the new association
_assoc = assocType(rrel = _rinstance, lrel = _linstance)
session.add(_assoc)
session.commit()
else:
if(_rinstance==None):
#_event = Event(ETitle=eventName, Edate='20.07.2019', EUrl='www.hallo.de')
_rinstance = __rinstance
if(_linstance==None):
#_organisation = Organisation(OName=organisationName, ODescription='Super Orga')
_linstance = __organisation
# create only the new association: missing events and/or organisations will be commited automatically
_assoc = assocType(rrel = _rinstance, lrel = _linstance)
session.add(_assoc)
session.commit()
commit_ltarget_rtarget(__organisation, __event, rcolumnAttribute, lcolumnAttribute, assocType)

View File

@ -1,22 +0,0 @@
import sqlalchemy
from sqlalchemy import Index
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, backref
from src.ORM_modules.base import Base, NgoBase
class Organisation(Base, NgoBase):
#__tablename__ = 'organisation'
OID = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
OName = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=False)
ODescription = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=True)
OUrl = sqlalchemy.Column(sqlalchemy.String(length=255), nullable=True)
OContact = sqlalchemy.Column(sqlalchemy.Integer)
#event = relationship('Event', secondary='TOrganisation_Events')
def getIndex(self):
return self.OID
__table_args__ = (Index('OName', "OName", "ODescription", "OUrl", "OContact"), )
def __repr__(self):
return ""

View File

@ -1,26 +0,0 @@
import sqlalchemy
from sqlalchemy import Index, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from src.ORM_modules.base import Base
from src.ORM_modules.Event import Event
from sqlalchemy.orm import relationship, backref
from src.ORM_modules.Organisation import Organisation
class Organisation_Events(Base):
__tablename__ = 'TOrganisation_Events'
#OEID = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
FK_EID = sqlalchemy.Column(sqlalchemy.Integer, ForeignKey("event.EID",onupdate="CASCADE", ondelete="CASCADE"), nullable=False, primary_key=True)
FK_OID = sqlalchemy.Column(sqlalchemy.Integer, ForeignKey("organisation.OID",onupdate="CASCADE", ondelete="CASCADE"), nullable=False, primary_key=True)
rrel = relationship(Event, backref=backref("event_assoc"))
lrel = relationship(Organisation, backref=backref("organisation_assoc"))
def __repr__(self):
return ""
@classmethod
def getLeftFK(cls):
return Organisation_Events.FK_OID
@classmethod
def getRightFK(cls):
return Organisation_Events.FK_EID

View File

@ -1,73 +0,0 @@
#check if event already exists
def commit_organisation_events(__organisation, __event):
#te = type(__event)
_event = session.query(Event).filter(Event.ETitle == __event.ETitle).first()
#check if organisation exists
_organisation = session.query(Organisation).filter(Organisation.OName == __organisation.OName).first()
_organisation_Events = None
# many to many:
# if both exist dont create either, but check association. => it's possible that an event is there + organisations is there but the event is created by another organisation
if ((_event != None) and (_organisation != None)):
_organisation_Events = session.query(Organisation_Events).filter(and_(Organisation_Events.FK_EID == _event.EID, Organisation_Events.FK_OID == _organisation.OID)).first()
if (_organisation_Events == None):
# create only the new association
_organisation_events = Organisation_Events(event = _event, organisation = _organisation)
session.add(_organisation_events)
session.commit()
else:
if(_event==None):
#_event = Event(ETitle=eventName, Edate='20.07.2019', EUrl='www.hallo.de')
_event = __event
if(_organisation==None):
#_organisation = Organisation(OName=organisationName, ODescription='Super Orga')
_organisation = __organisation
# create only the new association: missing events and/or organisations will be commited automatically
_organisation_events = Organisation_Events(event = _event, organisation = _organisation)
session.add(_organisation_events)
session.commit()
commit_organisation_events(__organisation, __event, rcolumn, rcolumn_target)
'''eventlist = []
organisationlist = []
organisation_events_list = []
_event1 = Event(ETitle='event1', Edate='20.07.2019', EUrl='www.hallo.de')
eventlist.append(_event1)
_organisation1 = Organisation(OName='Supger GMBH')
organisationlist.append(_organisation1)
organisation_events_list.append(Organisation_Events(event = _event1, organisation = _organisation1))
_event = Event(ETitle='event2', Edate='20.07.2019', EUrl='www.hallo.de')
eventlist.append(_event)
_organisation = Organisation(OName='ok CoKG')
organisationlist.append(_organisation)
organisation_events_list.append(Organisation_Events(event = _event, organisation = _organisation))
organisation_events_list.append(Organisation_Events(event = _event1, organisation = _organisation))
_event1 = Event(ETitle='event2', Edate='20.07.2019', EUrl='www.hallo.de')
_organisation = Organisation(OName='ok CoKG')
organisation_events_list.append(Organisation_Events(event = _event1, organisation = _organisation))
#organisation_events_list.append(Organisation_Events(FK_EID=_event1.EID, FK_OID=_organisation.OID))
# Add a user
#session.add_all(eventlist)
#session.add_all(organisationlist)
session.add_all(organisation_events_list)
session.commit()
'''
# Query the user
#our_user = session.query(User).filter_by(name='jesper').first()
#print('\nOur User:')
#print(our_user)
#print('Nick name in hex: {0}'.format(our_user.nickname.encode('utf-8')))

View File

@ -1,30 +0,0 @@
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.declarative import declared_attr
import sqlalchemy
from abc import ABCMeta, abstractmethod
Base = declarative_base()
class IInterface:
__metaclass__ = ABCMeta
@abstractmethod
def getIndex(self): raise NotImplementedError
class NgoBase(IInterface):
def __init__(self):
print(self)
@declared_attr
def __tablename__(cls):
return cls.__name__.lower()
#class TestClass(Base, NgoBase):
# EID = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
# def getIndex(self):
# return self.EID

View File

@ -1,72 +1,39 @@
# This file may be used to create an environment using:
# $ conda create --name <env> --file <this file>
# platform: win-64
appdirs=1.4.3=py37h28b3542_0
asn1crypto=0.24.0=py37_0
atomicwrites=1.3.0=py37_1
attrs=19.1.0=py37_1
automat=0.7.0=py37_0
bcrypt=3.1.6=py37he774522_0
beautifulsoup4=4.7.1=py37_1
ca-certificates=2019.5.15=0
certifi=2019.6.16=py37_1
cffi=1.12.3=py37h7a1dbc1_0
chardet=3.0.4=py37_1003
colorama=0.4.1=py37_0
constantly=15.1.0=py37h28b3542_0
cryptography=2.7=py37h7a1dbc1_0
cssselect=1.0.3=py37_0
hyperlink=19.0.0=py_0
idna=2.8=py37_0
importlib_metadata=0.19=py37_0
incremental=17.5.0=py37_0
libiconv=1.15=h1df5818_7
libprotobuf=3.6.0=h1a1b453_0
libxml2=2.9.9=h464c3ec_0
libxslt=1.1.33=h579f668_0
lxml=4.3.4=py37h1350720_0
more-itertools=7.2.0=py37_0
mysql-connector-c=6.1.11=h33f27b4_0
mysql-connector-python=8.0.16=py37h13ed8b8_0
mysqlclient=1.3.14=py37he774522_0
openssl=1.1.1c=he774522_1
packaging=19.0=py37_0
parsel=1.5.1=py37_0
pip=19.1.1=py37_0
pluggy=0.12.0=py_0
protobuf=3.6.0=py37he025d50_0
py=1.8.0=py37_0
pyasn1=0.4.6=py_0
pyasn1-modules=0.2.6=py37_0
pycparser=2.19=py37_0
pydispatcher=2.0.5=py37_1
pyhamcrest=1.9.0=py37_2
pyopenssl=19.0.0=py37_0
pyparsing=2.4.0=py_0
pysocks=1.7.0=py37_0
pytest=5.0.1=py37_0
pytest-runner=5.1=py_0
python=3.7.3=h8c8aaf0_1
pywin32=223=py37hfa6e2cd_1
queuelib=1.5.0=py37_0
requests=2.22.0=py37_0
scrapy=1.6.0=py37_0
service_identity=18.1.0=py37h28b3542_0
setuptools=41.0.1=py37_0
six=1.12.0=py37_0
soupsieve=1.9.2=py37_0
sqlalchemy=1.3.5=py37he774522_0
sqlite=3.29.0=he774522_0
twisted=19.2.0=py37he774522_0
urllib3=1.24.2=py37_0
vc=14.1=h0510ff6_4
vs2015_runtime=14.15.26706=h3a45250_4
w3lib=1.20.0=py37_0
wcwidth=0.1.7=py37_0
wheel=0.33.4=py37_0
win_inet_pton=1.1.0=py37_0
wincertstore=0.2=py37_0
zipp=0.5.2=py_0
zlib=1.2.11=h62dcd97_3
zope=1.0=py37_1
zope.interface=4.6.0=py37he774522_0

1
src/testtext.txt Normal file
View File

@ -0,0 +1 @@
erere