aboutsummaryrefslogtreecommitdiff
path: root/crawler/settings.py
blob: c0128a54f0acf0f900c95c17dd5b70ae96679485 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# -*- coding: utf-8 -*-

BOT_NAME = 'khanindex'

SPIDER_MODULES = ['crawler.nevrax.spiders', 'crawler.neodarznet.spiders']
#NEWSPIDER_MODULE = 'crawler.nevrax.spiders'

ROBOTSTXT_OBEY = True

DEPTH_LIMIT = 0

#ITEM_PIPELINES = {'crawler.nevrax.pipelines.NevraxPipeline': 0, 'crawler.neodarznet.pipelines.NeodarznetPipeline': 0}

LOG_LEVEL = 'WARNING'