diff options
| author | neodarz <neodarz@neodarz.net> | 2019-01-19 00:02:57 +0100 |
|---|---|---|
| committer | neodarz <neodarz@neodarz.net> | 2019-01-19 00:02:57 +0100 |
| commit | a3f01580faf6caee4abcc8e682567b87380857b9 (patch) | |
| tree | dbf5e0a9866b3aac7e7bb64f2eab8c005e1c28cd /crawler/settings.py | |
| parent | 073e919ef198a04da1e5ed28a7dfbc5d9681fc14 (diff) | |
| download | khanindexer-a3f01580faf6caee4abcc8e682567b87380857b9.tar.xz khanindexer-a3f01580faf6caee4abcc8e682567b87380857b9.zip | |
Add khanindex nevrax indexation
Diffstat (limited to '')
| -rw-r--r-- | crawler/settings.py | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/crawler/settings.py b/crawler/settings.py new file mode 100644 index 0000000..2ccdc11 --- /dev/null +++ b/crawler/settings.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +BOT_NAME = 'khanindex' + +SPIDER_MODULES = ['crawler.nevrax.spiders', 'crawler.neodarznet.spiders'] +#NEWSPIDER_MODULE = 'crawler.nevrax.spiders' + +ROBOTSTXT_OBEY = True + +DEPTH_LIMIT = 0 + +#ITEM_PIPELINES = {'crawler.nevrax.pipelines.NevraxPipeline': 0, 'crawler.neodarznet.pipelines.NeodarznetPipeline': 0} |