diff options
| author | neodarz <neodarz@neodarz.net> | 2019-01-10 23:54:18 +0100 |
|---|---|---|
| committer | neodarz <neodarz@neodarz.net> | 2019-01-10 23:54:18 +0100 |
| commit | 82746a2b76e5948333133bfe07c1042af2cd33b7 (patch) | |
| tree | 6a6302270472d079924d648644a2786ad115d200 /crawler/neodarznet/settings.py | |
| download | khanindexer-82746a2b76e5948333133bfe07c1042af2cd33b7.tar.xz khanindexer-82746a2b76e5948333133bfe07c1042af2cd33b7.zip | |
Initial commit
Diffstat (limited to '')
| -rw-r--r-- | crawler/neodarznet/settings.py | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/crawler/neodarznet/settings.py b/crawler/neodarznet/settings.py new file mode 100644 index 0000000..8d65b09 --- /dev/null +++ b/crawler/neodarznet/settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +BOT_NAME = 'neodarznet' + +SPIDER_MODULES = ['crawler.neodarznet.spiders'] +NEWSPIDER_MODULE = 'crawler.neodarznet.spiders' + +ROBOTSTXT_OBEY = True + +DEPTH_LIMIT = 0 |