Add khanindex nevrax indexation

1 files changed, 6 insertions, 4 deletions

diff --git a/app.py b/app.py
index 31f338d..66ca6c4 100644
--- a/app.py
+++ b/app.py
@@ -1,14 +1,13 @@
 import scrapy
 import sys
 from scrapy.crawler import CrawlerProcess
+from scrapy import spiderloader
 from scrapy.utils.project import get_project_settings
 from flask import Flask, request, jsonify
 import json
 
 from sphinx import sphinx
 
-from crawler.neodarznet.spiders.scrape import ScrapSpider
-
 from database.models import Page, db
 import config
 
@@ -34,8 +33,11 @@ def search():
 def crawl():
     try:
         db.create_tables(Page.__subclasses__())
-        process = CrawlerProcess(get_project_settings())
-        process.crawl(ScrapSpider)
+        settings = get_project_settings()
+        process = CrawlerProcess(settings)
+        spiders = spiderloader.SpiderLoader.from_settings(settings)
+        for spider in spiders.list():
+            process.crawl(spider)
         process.start()
     except Exception as e:
         print(e)