crawler/neodarznet/settings.py


1
2
3
4
5
6
7
8
9
10
11
12

# -*- coding: utf-8 -*-

BOT_NAME = 'neodarznet'

SPIDER_MODULES = ['crawler.neodarznet.spiders']
NEWSPIDER_MODULE = 'crawler.neodarznet.spiders'

ROBOTSTXT_OBEY = True

DEPTH_LIMIT = 0

ITEM_PIPELINES = {'crawler.neodarznet.pipelines.NeodarznetPipeline': 0}