crawler/settings.py


1
2
3
4
5
6
7
8
9
10
11
12

# -*- coding: utf-8 -*-

BOT_NAME = 'khanindex'

SPIDER_MODULES = ['crawler.nevrax.spiders', 'crawler.neodarznet.spiders']
#NEWSPIDER_MODULE = 'crawler.nevrax.spiders'

ROBOTSTXT_OBEY = True

DEPTH_LIMIT = 0

#ITEM_PIPELINES = {'crawler.nevrax.pipelines.NevraxPipeline': 0, 'crawler.neodarznet.pipelines.NeodarznetPipeline': 0}