aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.md5
-rw-r--r--app.py14
-rw-r--r--config.py6
-rw-r--r--database/models.py7
-rw-r--r--docker-compose.yml7
5 files changed, 28 insertions, 11 deletions
diff --git a/README.md b/README.md
index 6f03880..1944f82 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,8 @@ The database is in the sqlite file `khanindexer.db` at the root of the project.
# Testing
-If you just want to test and don't want to install a MySQL (or MariaDB)
-database but have Docker installed, juste use the `docker-compose.yml`.
+If you just want to test and don't want to install a PostgreSQL database
+but have Docker installed, juste use the `docker-compose.yml`.
+
This is only for test, don't use this shit on production (the docker-compose
file)!
diff --git a/app.py b/app.py
index 281f932..ee04e3d 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,8 @@
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
+from psycopg2 import connect
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
from crawler.neodarznet.spiders.scrape import ScrapSpider
@@ -10,7 +12,17 @@ import config
process = CrawlerProcess(get_project_settings())
def main():
- db.create_tables([Page])
+
+ try:
+ db.create_tables([Page])
+ except:
+ con = connect(user=config.DB_USER, host=config.DB_HOST, password=config.DB_PASS)
+ con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+ cur = con.cursor()
+ cur.execute('CREATE DATABASE '+config.DB+';')
+ cur.close()
+ con.close()
+ db.create_tables([Page])
process.crawl(ScrapSpider)
process.start()
diff --git a/config.py b/config.py
index 58de78f..9df6ddc 100644
--- a/config.py
+++ b/config.py
@@ -2,4 +2,8 @@ from os import path
APP_DIR = path.dirname(__file__)
-DATABASE = '%s' % path.join(APP_DIR, 'khanindexer.db')
+DB = "khanindexer"
+DB_HOST = "127.0.0.1"
+DB_PORT = "5432"
+DB_USER = "root"
+DB_PASS = "root"
diff --git a/database/models.py b/database/models.py
index c731a08..2f9528a 100644
--- a/database/models.py
+++ b/database/models.py
@@ -1,9 +1,8 @@
-from peewee import Model, CharField, SqliteDatabase
+from peewee import Model, CharField, TextField, PostgresqlDatabase
import config
-db = SqliteDatabase(config.DATABASE)
-db.connect()
+db = PostgresqlDatabase(config.DB, host=config.DB_HOST, port=config.DB_PORT, user=config.DB_USER, password=config.DB_PASS, autocommit=True, autorollback=True)
class Page(Model):
"""
@@ -11,7 +10,7 @@ class Page(Model):
"""
url = CharField()
title = CharField(null=True)
- content = CharField(null=True)
+ content = TextField(null=True)
class Meta:
database = db
diff --git a/docker-compose.yml b/docker-compose.yml
index b2aca59..74f82b5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,8 +2,9 @@ version: "3"
services:
mysql:
- image: mariadb
+ image: postgres
environment:
- MYSQL_ROOT_PASSWORD: root
+ POSTGRES_USER: root
+ POSTGRES_PASSWORD: root
ports:
- - "3306:3306"
+ - "5432:5432"