aboutsummaryrefslogtreecommitdiff
path: root/crawler/nevrax
diff options
context:
space:
mode:
authorneodarz <neodarz@neodarz.net>2019-02-06 00:02:31 +0100
committerneodarz <neodarz@neodarz.net>2019-02-06 00:02:31 +0100
commit74dbf4defed8ae348a327e4674d917b3dd869713 (patch)
tree36cf3c0f45b443616eccf4473e6c02e15e6a2347 /crawler/nevrax
parentad6212da067fdc05a8564e79943692fd9d466110 (diff)
downloadkhanindexer-74dbf4defed8ae348a327e4674d917b3dd869713.tar.xz
khanindexer-74dbf4defed8ae348a327e4674d917b3dd869713.zip
Add date when page are crawled
Diffstat (limited to 'crawler/nevrax')
-rw-r--r--crawler/nevrax/spiders/scrape.py4
1 files changed, 3 insertions, 1 deletions
diff --git a/crawler/nevrax/spiders/scrape.py b/crawler/nevrax/spiders/scrape.py
index 785ec3f..d27aecf 100644
--- a/crawler/nevrax/spiders/scrape.py
+++ b/crawler/nevrax/spiders/scrape.py
@@ -5,6 +5,7 @@ from scrapy.linkextractors import LinkExtractor
from scrapy import Selector
import config
+import datetime
class NevraxSpider(CrawlSpider):
name = "nevrax"
@@ -42,5 +43,6 @@ class NevraxSpider(CrawlSpider):
'url': response.url,
'title': response.css('title::text').extract_first(),
'content': ''.join(sel.select("//body//text()").extract()).strip(),
- 'content_length': len(response.body)
+ 'content_length': len(response.body),
+ 'date_updated': datetime.datetime.now()
}