From 07cc1fdc7e79468a6e8d9b53048712b652f8da84 Mon Sep 17 00:00:00 2001 From: neodarz Date: Sun, 13 Jan 2019 18:34:54 +0100 Subject: Migrate to PostgreSQL --- app.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'app.py') diff --git a/app.py b/app.py index 281f932..ee04e3d 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,8 @@ import scrapy from scrapy.crawler import CrawlerProcess from scrapy.utils.project import get_project_settings +from psycopg2 import connect +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from crawler.neodarznet.spiders.scrape import ScrapSpider @@ -10,7 +12,17 @@ import config process = CrawlerProcess(get_project_settings()) def main(): - db.create_tables([Page]) + + try: + db.create_tables([Page]) + except: + con = connect(user=config.DB_USER, host=config.DB_HOST, password=config.DB_PASS) + con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + cur = con.cursor() + cur.execute('CREATE DATABASE '+config.DB+';') + cur.close() + con.close() + db.create_tables([Page]) process.crawl(ScrapSpider) process.start() -- cgit v1.2.1