From ad6212da067fdc05a8564e79943692fd9d466110 Mon Sep 17 00:00:00 2001
From: neodarz <neodarz@neodarz.net>
Date: Sun, 3 Feb 2019 20:02:18 +0100
Subject: Add ability to update a page in db if updated

---
 crawler/nevrax/spiders/scrape.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'crawler/nevrax/spiders')

diff --git a/crawler/nevrax/spiders/scrape.py b/crawler/nevrax/spiders/scrape.py
index 8a7b8ec..785ec3f 100644
--- a/crawler/nevrax/spiders/scrape.py
+++ b/crawler/nevrax/spiders/scrape.py
@@ -41,5 +41,6 @@ class NevraxSpider(CrawlSpider):
         yield {
                 'url': response.url,
                 'title': response.css('title::text').extract_first(),
-                'content': ''.join(sel.select("//body//text()").extract()).strip()
+                'content': ''.join(sel.select("//body//text()").extract()).strip(),
+                'content_length': len(response.body)
         }
-- 
cgit v1.2.1