diff options
| author | neodarz <neodarz@neodarz.net> | 2019-02-03 20:02:18 +0100 |
|---|---|---|
| committer | neodarz <neodarz@neodarz.net> | 2019-02-03 20:02:18 +0100 |
| commit | ad6212da067fdc05a8564e79943692fd9d466110 (patch) | |
| tree | 7d83c4d90e7b0bb0260cf6dce39ea132761ebea8 /crawler/nevrax/spiders | |
| parent | c66ea291fe937a9b1321f5b3d417669b2aafa43b (diff) | |
| download | khanindexer-ad6212da067fdc05a8564e79943692fd9d466110.tar.xz khanindexer-ad6212da067fdc05a8564e79943692fd9d466110.zip | |
Add ability to update a page in db if updated
Diffstat (limited to 'crawler/nevrax/spiders')
| -rw-r--r-- | crawler/nevrax/spiders/scrape.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/crawler/nevrax/spiders/scrape.py b/crawler/nevrax/spiders/scrape.py index 8a7b8ec..785ec3f 100644 --- a/crawler/nevrax/spiders/scrape.py +++ b/crawler/nevrax/spiders/scrape.py @@ -41,5 +41,6 @@ class NevraxSpider(CrawlSpider): yield { 'url': response.url, 'title': response.css('title::text').extract_first(), - 'content': ''.join(sel.select("//body//text()").extract()).strip() + 'content': ''.join(sel.select("//body//text()").extract()).strip(), + 'content_length': len(response.body) } |