Add first implemention of a cron like functionHEADmaster

1 files changed, 43 insertions, 5 deletions

diff --git a/app.py b/app.py
index 9c1c302..744b697 100644
--- a/app.py
+++ b/app.py
@@ -1,17 +1,27 @@
 import scrapy
 import sys
-from scrapy.crawler import CrawlerProcess
+import os
+import time
+from scrapy.crawler import CrawlerProcess, CrawlerRunner
 from scrapy import spiderloader
 from scrapy.utils.project import get_project_settings
+from crochet import setup
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers import cron
 
 from database.models import Page, db
+import config
 
-def crawl(the_spider=""):
+def crawl(the_spider="", cron=False):
     try:
         db.create_tables(Page.__subclasses__())
         settings = get_project_settings()
-        process = CrawlerProcess(settings)
         spiders = spiderloader.SpiderLoader.from_settings(settings)
+        if cron:
+            setup()
+            process = CrawlerRunner(settings)
+        else:
+            process = CrawlerProcess(settings)
 
         if the_spider == "":
             for spider in spiders.list():
@@ -27,9 +37,10 @@ def crawl(the_spider=""):
                     print("    "+spider)
             sys.exit(1)
 
-        process.start()
+        if not cron:
+            process.start()
     except Exception as e:
-        print(e)
+        print("Error:"+str(e))
 
 def update():
     try:
@@ -43,6 +54,31 @@ def update():
     except Exception as e:
         print(e)
 
+def cron_scheduling():
+    try:
+        db.create_tables(Page.__subclasses__())
+
+        scheduler = BackgroundScheduler()
+        scheduler.add_jobstore('sqlalchemy', url=config.CRON_DB)
+        variables = ['minute', 'hour', 'month', 'day_of_week', 'crawler', 'id']
+        with open(config.CRON_CONF) as f:
+            for line in f:
+                if not line.startswith('#'):
+                    datas = list(filter(lambda x: x != "", line.rstrip().split(" ")))
+                    cron_table = dict(zip(variables, datas))
+                    scheduler.add_job(crawl, 'cron', args=[cron_table['crawler'], 'cron'], minute=cron_table['minute'], hour=cron_table['hour'], month=cron_table['month'], day_of_week=cron_table['day_of_week'], id=cron_table['id'], replace_existing=True)
+        print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
+
+        scheduler.start()
+
+        while True:
+            time.sleep(1)
+
+    except Exception as e:
+        print(e)
+
+
+
 def show_help():
     print("Launch all crawler               => "+str(sys.argv[0])+" crawl")
     print("Update all page already crawlerd => "+str(sys.argv[0])+" update")
@@ -54,6 +90,8 @@ if __name__ == '__main__':
             crawl()
         elif sys.argv[1] == "update":
             update()
+        elif sys.argv[1] == "cron":
+            cron_scheduling()
         else:
             show_help()
     elif len(sys.argv) == 3: