diff options
Diffstat (limited to '')
| -rw-r--r-- | crawler/neodarznet/spiders/scrape.py | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/crawler/neodarznet/spiders/scrape.py b/crawler/neodarznet/spiders/scrape.py index 0f54c4a..bd97067 100644 --- a/crawler/neodarznet/spiders/scrape.py +++ b/crawler/neodarznet/spiders/scrape.py @@ -4,6 +4,8 @@ from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor from scrapy import Selector +import datetime + class NeodarznetSpider(CrawlSpider): name = "neodarznet" custom_settings = { @@ -40,5 +42,6 @@ class NeodarznetSpider(CrawlSpider): 'url': response.url, 'title': response.css('title::text').extract_first(), 'content': ''.join(sel.select("//div[@class='bodya']//text()").extract()).strip(), - 'content_length': len(response.body) + 'content_length': len(response.body), + 'date_updated': datetime.datetime.now() } |