From ad6212da067fdc05a8564e79943692fd9d466110 Mon Sep 17 00:00:00 2001 From: neodarz Date: Sun, 3 Feb 2019 20:02:18 +0100 Subject: Add ability to update a page in db if updated --- crawler/nevrax/pipelines.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'crawler/nevrax/pipelines.py') diff --git a/crawler/nevrax/pipelines.py b/crawler/nevrax/pipelines.py index 775d5df..dbbb782 100644 --- a/crawler/nevrax/pipelines.py +++ b/crawler/nevrax/pipelines.py @@ -9,8 +9,9 @@ class NevraxPipeline(object): def process_item(self, item, spider): try: page = Nevrax.get(Nevrax.url == item['url']) - q = Nevrax.update(**item).where(Nevrax.url == item['url']) - q.execute() + if page.content_length != item['content_length']: + q = Nevrax.update(**item).where(Nevrax.url == item['url']) + q.execute() logging.info("Update item {}".format(page)) except Nevrax.DoesNotExist: page = Nevrax.create(**item) -- cgit v1.2.1