diff options
| -rw-r--r-- | README.md | 5 | ||||
| -rw-r--r-- | app.py | 14 | ||||
| -rw-r--r-- | config.py | 6 | ||||
| -rw-r--r-- | database/models.py | 7 | ||||
| -rw-r--r-- | docker-compose.yml | 7 |
5 files changed, 28 insertions, 11 deletions
@@ -21,7 +21,8 @@ The database is in the sqlite file `khanindexer.db` at the root of the project. # Testing -If you just want to test and don't want to install a MySQL (or MariaDB) -database but have Docker installed, juste use the `docker-compose.yml`. +If you just want to test and don't want to install a PostgreSQL database +but have Docker installed, juste use the `docker-compose.yml`. + This is only for test, don't use this shit on production (the docker-compose file)! @@ -1,6 +1,8 @@ import scrapy from scrapy.crawler import CrawlerProcess from scrapy.utils.project import get_project_settings +from psycopg2 import connect +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from crawler.neodarznet.spiders.scrape import ScrapSpider @@ -10,7 +12,17 @@ import config process = CrawlerProcess(get_project_settings()) def main(): - db.create_tables([Page]) + + try: + db.create_tables([Page]) + except: + con = connect(user=config.DB_USER, host=config.DB_HOST, password=config.DB_PASS) + con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + cur = con.cursor() + cur.execute('CREATE DATABASE '+config.DB+';') + cur.close() + con.close() + db.create_tables([Page]) process.crawl(ScrapSpider) process.start() @@ -2,4 +2,8 @@ from os import path APP_DIR = path.dirname(__file__) -DATABASE = '%s' % path.join(APP_DIR, 'khanindexer.db') +DB = "khanindexer" +DB_HOST = "127.0.0.1" +DB_PORT = "5432" +DB_USER = "root" +DB_PASS = "root" diff --git a/database/models.py b/database/models.py index c731a08..2f9528a 100644 --- a/database/models.py +++ b/database/models.py @@ -1,9 +1,8 @@ -from peewee import Model, CharField, SqliteDatabase +from peewee import Model, CharField, TextField, PostgresqlDatabase import config -db = SqliteDatabase(config.DATABASE) -db.connect() +db = PostgresqlDatabase(config.DB, host=config.DB_HOST, port=config.DB_PORT, user=config.DB_USER, password=config.DB_PASS, autocommit=True, autorollback=True) class Page(Model): """ @@ -11,7 +10,7 @@ class Page(Model): """ url = CharField() title = CharField(null=True) - content = CharField(null=True) + content = TextField(null=True) class Meta: database = db diff --git a/docker-compose.yml b/docker-compose.yml index b2aca59..74f82b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,9 @@ version: "3" services: mysql: - image: mariadb + image: postgres environment: - MYSQL_ROOT_PASSWORD: root + POSTGRES_USER: root + POSTGRES_PASSWORD: root ports: - - "3306:3306" + - "5432:5432" |