From a3f01580faf6caee4abcc8e682567b87380857b9 Mon Sep 17 00:00:00 2001 From: neodarz Date: Sat, 19 Jan 2019 00:02:57 +0100 Subject: Add khanindex nevrax indexation --- app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'app.py') diff --git a/app.py b/app.py index 31f338d..66ca6c4 100644 --- a/app.py +++ b/app.py @@ -1,14 +1,13 @@ import scrapy import sys from scrapy.crawler import CrawlerProcess +from scrapy import spiderloader from scrapy.utils.project import get_project_settings from flask import Flask, request, jsonify import json from sphinx import sphinx -from crawler.neodarznet.spiders.scrape import ScrapSpider - from database.models import Page, db import config @@ -34,8 +33,11 @@ def search(): def crawl(): try: db.create_tables(Page.__subclasses__()) - process = CrawlerProcess(get_project_settings()) - process.crawl(ScrapSpider) + settings = get_project_settings() + process = CrawlerProcess(settings) + spiders = spiderloader.SpiderLoader.from_settings(settings) + for spider in spiders.list(): + process.crawl(spider) process.start() except Exception as e: print(e) -- cgit v1.2.1