add log support for scrapy

ResolveWang · ResolveWang · commit fb0f702a0832 · 2018-03-03T15:45:45.000+08:00
diff --git a/config/rules.py b/config/rules.py
@@ -362,6 +362,13 @@
         'internal': 20,
         'enable': 1,
     },
+    {
+        'name': 'zhihu',
+        'task_queue': TEMP_ZHIHU_QUEUE,
+        'resource': VALIDATED_ZHIHU_QUEUE,
+        'internal': 20,
+        'enable': 1,
+    },
 ]
 
 # crawlers will fetch tasks from the following queues
diff --git a/config/settings.py b/config/settings.py
@@ -44,6 +44,7 @@
 
 # scrapy log settings
 LOG_LEVEL = 'DEBUG'
+LOG_FILE = 'logs/haipproxy.log'
 
 
 #####################################################################
diff --git a/crawler/redis_spiders.py b/crawler/redis_spiders.py
@@ -7,12 +7,15 @@
 from scrapy.spiders import (
     Spider, CrawlSpider)
 from scrapy_splash import SplashRequest
+from scrapy.utils.log import configure_logging
 
 from logger import crawler_logger
 from utils import get_redis_conn
 from config.settings import (
     VALIDATOR_FEED_SIZE, SPIDER_FEED_SIZE)
 
+
+configure_logging(install_root_handler=True)
 __all__ = ['RedisSpider', 'RedisAjaxSpider',
            'RedisCrawlSpider', 'ValidatorRedisSpider']
 
diff --git a/logger/__init__.py b/logger/__init__.py
@@ -12,7 +12,7 @@
 if not os.path.exists(log_dir):
     os.mkdir(log_dir)
 
-log_path = os.path.join(log_dir, 'haiproxy.log')
+log_path = os.path.join(log_dir, 'haipproxy.log')
 
 log_config = {
     'version': 1.0,
diff --git a/scheduler/scheduler.py b/scheduler/scheduler.py
@@ -113,7 +113,7 @@ def schedule_task_with_lock(self, task):
                 pipe.lpush(task_queue, *urls)
                 pipe.hset(TIMER_RECORDER, task_name, now)
                 pipe.execute()
-                scheduler_logger('crawler task {} has been stored into redis successfully'.format(task_name))
+                scheduler_logger.info('crawler task {} has been stored into redis successfully'.format(task_name))
                 return True
             else:
                 return None
@@ -146,13 +146,13 @@ def schedule_task_with_lock(self, task):
             r, proxies = pipe.execute()
             if not r or (now - int(r.decode('utf-8'))) >= internal * 60:
                 if not proxies:
-                    scheduler_logger('fetched no proxies from task {}'.format(task_name))
+                    scheduler_logger.warning('fetched no proxies from task {}'.format(task_name))
                     return None
 
                 pipe.sadd(task_queue, *proxies)
                 pipe.hset(TIMER_RECORDER, task_name, now)
                 pipe.execute()
-                scheduler_logger('validator task {} has been stored into redis successfully'.format(task_name))
+                scheduler_logger.info('validator task {} has been stored into redis successfully'.format(task_name))
                 return True
             else:
                 return None
@@ -165,6 +165,7 @@ def schedule_task_with_lock(self, task):
 @click.argument('task_queues', nargs=-1)
 def scheduler_start(usage, task_queues):
     """Start specified scheduler."""
+    scheduler_logger.info('{} scheduler is starting...'.format(usage))
     default_tasks = CRWALER_TASKS if usage == 'crawler' else VALIDATOR_TASKS
     default_allow_tasks = DEFAULT_CRAWLER_TASKS if usage == 'crawler' else DEFAULT_VALIDATORS_TASKS
     maps = CRAWLER_TASK_MAPS if usage == 'crawler' else TEMP_TASK_MAPS
@@ -177,7 +178,7 @@ def scheduler_start(usage, task_queues):
         for task_queue in task_queues:
             allow_task_queue = maps.get(task_queue)
             if not allow_task_queue:
-                scheduler_logger('scheduler task {} is invalid task, the allowed tasks are {}'.format(
+                scheduler_logger.warning('scheduler task {} is an invalid task, the allowed tasks are {}'.format(
                     task_queue, list(maps.keys())))
                 continue
             scheduler.task_queues.append(allow_task_queue)
@@ -207,7 +208,7 @@ def crawler_start(usage, tasks):
                     spiders.append(case.spider)
                     break
             else:
-                crawler_logger.warning('spider task {} is invalid task, the allowed tasks are {}'.format(
+                crawler_logger.warning('spider task {} is an invalid task, the allowed tasks are {}'.format(
                     task, list(maps.keys())))
     if not spiders:
         crawler_logger.warning('no spider starts up, please check your task input')
