blob: 431c2b0c38233d2f7b4ccc490fa89544f63b92cd (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
import scrapy
import sys
from scrapy.crawler import CrawlerProcess
from scrapy import spiderloader
from scrapy.utils.project import get_project_settings
from database.models import Page, db
def main():
try:
db.create_tables(Page.__subclasses__())
settings = get_project_settings()
process = CrawlerProcess(settings)
spiders = spiderloader.SpiderLoader.from_settings(settings)
for spider in spiders.list():
process.crawl(spider)
process.start()
except Exception as e:
print(e)
if __name__ == '__main__':
main()
|
