fix bugs for logger

ResolveWang · ResolveWang · commit 67d693879f64 · 2018-03-03T15:14:22.000+08:00
diff --git a/crawler/redis_spiders.py b/crawler/redis_spiders.py
@@ -8,8 +8,7 @@
     Spider, CrawlSpider)
 from scrapy_splash import SplashRequest
 
-from logger import (
-    crawler_logger, validator_logger)
+from logger import crawler_logger
 from utils import get_redis_conn
 from config.settings import (
     VALIDATOR_FEED_SIZE, SPIDER_FEED_SIZE)
@@ -49,7 +48,7 @@ def next_requests(self):
                 yield req
                 found += 1
 
-        crawler_logger.debug('Read {} requests from {}'.format(found, self.task_queue))
+        crawler_logger.info('Read {} requests from {}'.format(found, self.task_queue))
 
     def schedule_next_requests(self):
         for req in self.next_requests():
@@ -97,7 +96,7 @@ def next_requests(self):
                 yield req
                 found += 1
 
-        crawler_logger.debug('Read {} requests from {}'.format(found, self.task_queue))
+        crawler_logger.info('Read {} requests from {}'.format(found, self.task_queue))
 
 
 class ValidatorRedisSpider(RedisSpider):
@@ -122,7 +121,7 @@ def next_requests_process(self, task_queue):
                               callback=self.parse, errback=self.parse_error)
                 yield req
                 found += 1
-        validator_logger.debug('Read {} ip proxies from {}'.format(found, task_queue))
+        crawler_logger.info('Read {} ip proxies from {}'.format(found, task_queue))
 
     def parse_error(self, failure):
         raise NotImplementedError
diff --git a/crawler/validators/base.py b/crawler/validators/base.py
@@ -6,7 +6,7 @@
 from twisted.internet.error import (
     TimeoutError, TCPTimedOutError)
 
-from logger import validator_logger
+from logger import crawler_logger
 from ..items import (
     ProxyScoreItem, ProxyVerifiedTimeItem,
     ProxySpeedItem)
@@ -58,7 +58,7 @@ def is_transparent(self, response):
     def parse_error(self, failure):
         request = failure.request
         proxy = request.meta.get('proxy')
-        validator_logger.error('proxy {} has been failed,{} is raised'.format(proxy, failure))
+        crawler_logger.error('proxy {} has been failed,{} is raised'.format(proxy, failure))
         if failure.check(TimeoutError, TCPTimedOutError):
             decr = -1
         else:
diff --git a/logger/__init__.py b/logger/__init__.py
@@ -56,7 +56,7 @@
         },
         'other_logger': {
             'handlers': ['file'],
-            'level': 'info',
+            'level': 'INFO',
         }
     }
 }
diff --git a/scheduler/scheduler.py b/scheduler/scheduler.py
@@ -205,6 +205,7 @@ def crawler_start(usage, tasks):
             for case in cases:
                 if case.check(task, maps):
                     spiders.append(case.spider)
+                    break
             else:
                 crawler_logger.warning('spider task {} is invalid task, the allowed tasks are {}'.format(
                     task, list(maps.keys())))

Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@`
`56`	`56`	`},`
`57`	`57`	`'other_logger': {`
`58`	`58`	`'handlers': ['file'],`
`59`		`- 'level': 'info',`
	`59`	`+ 'level': 'INFO',`
`60`	`60`	`}`
`61`	`61`	`}`
`62`	`62`	`}`