diff options
| author | neodarz <neodarz@neodarz.net> | 2019-02-06 00:02:31 +0100 |
|---|---|---|
| committer | neodarz <neodarz@neodarz.net> | 2019-02-06 00:02:31 +0100 |
| commit | 74dbf4defed8ae348a327e4674d917b3dd869713 (patch) | |
| tree | 36cf3c0f45b443616eccf4473e6c02e15e6a2347 /crawler/nevrax/spiders | |
| parent | ad6212da067fdc05a8564e79943692fd9d466110 (diff) | |
| download | khanindexer-74dbf4defed8ae348a327e4674d917b3dd869713.tar.xz khanindexer-74dbf4defed8ae348a327e4674d917b3dd869713.zip | |
Add date when page are crawled
Diffstat (limited to 'crawler/nevrax/spiders')
| -rw-r--r-- | crawler/nevrax/spiders/scrape.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/crawler/nevrax/spiders/scrape.py b/crawler/nevrax/spiders/scrape.py index 785ec3f..d27aecf 100644 --- a/crawler/nevrax/spiders/scrape.py +++ b/crawler/nevrax/spiders/scrape.py @@ -5,6 +5,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy import Selector import config +import datetime class NevraxSpider(CrawlSpider): name = "nevrax" @@ -42,5 +43,6 @@ class NevraxSpider(CrawlSpider): 'url': response.url, 'title': response.css('title::text').extract_first(), 'content': ''.join(sel.select("//body//text()").extract()).strip(), - 'content_length': len(response.body) + 'content_length': len(response.body), + 'date_updated': datetime.datetime.now() } |