aboutsummaryrefslogtreecommitdiff
path: root/crawler
diff options
context:
space:
mode:
Diffstat (limited to 'crawler')
-rw-r--r--crawler/neodarznet/spiders/scrape.py5
-rw-r--r--crawler/nevrax/spiders/scrape.py4
2 files changed, 7 insertions, 2 deletions
diff --git a/crawler/neodarznet/spiders/scrape.py b/crawler/neodarznet/spiders/scrape.py
index 0f54c4a..bd97067 100644
--- a/crawler/neodarznet/spiders/scrape.py
+++ b/crawler/neodarznet/spiders/scrape.py
@@ -4,6 +4,8 @@ from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy import Selector
+import datetime
+
class NeodarznetSpider(CrawlSpider):
name = "neodarznet"
custom_settings = {
@@ -40,5 +42,6 @@ class NeodarznetSpider(CrawlSpider):
'url': response.url,
'title': response.css('title::text').extract_first(),
'content': ''.join(sel.select("//div[@class='bodya']//text()").extract()).strip(),
- 'content_length': len(response.body)
+ 'content_length': len(response.body),
+ 'date_updated': datetime.datetime.now()
}
diff --git a/crawler/nevrax/spiders/scrape.py b/crawler/nevrax/spiders/scrape.py
index 785ec3f..d27aecf 100644
--- a/crawler/nevrax/spiders/scrape.py
+++ b/crawler/nevrax/spiders/scrape.py
@@ -5,6 +5,7 @@ from scrapy.linkextractors import LinkExtractor
from scrapy import Selector
import config
+import datetime
class NevraxSpider(CrawlSpider):
name = "nevrax"
@@ -42,5 +43,6 @@ class NevraxSpider(CrawlSpider):
'url': response.url,
'title': response.css('title::text').extract_first(),
'content': ''.join(sel.select("//body//text()").extract()).strip(),
- 'content_length': len(response.body)
+ 'content_length': len(response.body),
+ 'date_updated': datetime.datetime.now()
}