Migrate to PostgreSQL

5 files changed, 28 insertions, 11 deletions

diff --git a/README.md b/README.md
index 6f03880..1944f82 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,8 @@ The database is in the sqlite file `khanindexer.db` at the root of the project.
 
 # Testing
 
-If you just want to test and don't want to install a MySQL (or MariaDB)
-database but have Docker installed, juste use the `docker-compose.yml`.
+If you just want to test and don't want to install a PostgreSQL database
+but have Docker installed, juste use the `docker-compose.yml`.
+
 This is only for test, don't use this shit on production (the docker-compose
 file)!
diff --git a/app.py b/app.py
index 281f932..ee04e3d 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,8 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.project import get_project_settings
+from psycopg2 import connect
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
 from crawler.neodarznet.spiders.scrape import ScrapSpider
 
@@ -10,7 +12,17 @@ import config
 process = CrawlerProcess(get_project_settings())
 
 def main():
-    db.create_tables([Page])
+
+    try:
+        db.create_tables([Page])
+    except:
+        con = connect(user=config.DB_USER, host=config.DB_HOST, password=config.DB_PASS)
+        con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+        cur = con.cursor()
+        cur.execute('CREATE DATABASE '+config.DB+';')
+        cur.close()
+        con.close()
+        db.create_tables([Page])
 
     process.crawl(ScrapSpider)
     process.start()
diff --git a/config.py b/config.py
index 58de78f..9df6ddc 100644
--- a/config.py
+++ b/config.py
@@ -2,4 +2,8 @@ from os import path
 
 APP_DIR = path.dirname(__file__)
 
-DATABASE = '%s' % path.join(APP_DIR, 'khanindexer.db')
+DB = "khanindexer"
+DB_HOST = "127.0.0.1"
+DB_PORT = "5432"
+DB_USER = "root"
+DB_PASS = "root"
diff --git a/database/models.py b/database/models.py
index c731a08..2f9528a 100644
--- a/database/models.py
+++ b/database/models.py
@@ -1,9 +1,8 @@
-from peewee import Model, CharField, SqliteDatabase
+from peewee import Model, CharField, TextField, PostgresqlDatabase
 
 import config
 
-db = SqliteDatabase(config.DATABASE)
-db.connect()
+db = PostgresqlDatabase(config.DB, host=config.DB_HOST, port=config.DB_PORT, user=config.DB_USER, password=config.DB_PASS, autocommit=True, autorollback=True)
 
 class Page(Model):
     """
@@ -11,7 +10,7 @@ class Page(Model):
     """
     url = CharField()
     title = CharField(null=True)
-    content = CharField(null=True)
+    content = TextField(null=True)
 
     class Meta:
         database = db
diff --git a/docker-compose.yml b/docker-compose.yml
index b2aca59..74f82b5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,8 +2,9 @@ version: "3"
 
 services:
     mysql:
-        image: mariadb
+        image: postgres
         environment:
-            MYSQL_ROOT_PASSWORD: root
+            POSTGRES_USER: root
+            POSTGRES_PASSWORD: root
         ports:
-            - "3306:3306"
+            - "5432:5432"