import scrapy
import sys
from scrapy.crawler import CrawlerProcess
from scrapy import spiderloader
from scrapy.utils.project import get_project_settings

from database.models import Page, db

def main():
    try:
        db.create_tables(Page.__subclasses__())
        settings = get_project_settings()
        process = CrawlerProcess(settings)
        spiders = spiderloader.SpiderLoader.from_settings(settings)
        for spider in spiders.list():
            process.crawl(spider)
        process.start()
    except Exception as e:
        print(e)

if __name__ == '__main__':
    main()