Add ability to update url who are one week old and content modified

1 files changed, 29 insertions, 3 deletions

diff --git a/app.py b/app.py
index 431c2b0..ad44a14 100644
--- a/app.py
+++ b/app.py
@@ -6,17 +6,43 @@ from scrapy.utils.project import get_project_settings
 
 from database.models import Page, db
 
-def main():
+def crawl():
     try:
         db.create_tables(Page.__subclasses__())
         settings = get_project_settings()
         process = CrawlerProcess(settings)
         spiders = spiderloader.SpiderLoader.from_settings(settings)
         for spider in spiders.list():
-            process.crawl(spider)
+            if "crawl" in spider:
+                process.crawl(spider)
         process.start()
     except Exception as e:
         print(e)
 
+def update():
+    try:
+        settings = get_project_settings()
+        process = CrawlerProcess(settings)
+        spiders = spiderloader.SpiderLoader.from_settings(settings)
+        for spider in spiders.list():
+            if "update" in spider:
+                process.crawl(spider)
+        process.start()
+    except Exception as e:
+        print(e)
+
+def show_help():
+    print("Launch all crawler               => "+str(sys.argv[0])+" crawl")
+    print("Update all page already crawlerd => "+str(sys.argv[0])+" update")
+
 if __name__ == '__main__':
-    main()
+    #main()
+    if len(sys.argv) == 2:
+        if sys.argv[1] == "crawl":
+            crawl()
+        elif sys.argv[1] == "update":
+            update()
+        else:
+            show_help()
+    else:
+        show_help()