Skip to content

Commit e0eca49

Browse files
bobobo80bobobo80
authored
Merge pull request jhao104#2 from jhao104/master
New update
2 parents c2bea0e + 728a282 commit e0eca49

24 files changed

+693
-263
lines changed

‎Api/ProxyApi.py

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@
1515

1616
import sys
1717

18+
sys.path.append('../')
19+
1820
from flask import Flask, jsonify, request
21+
from Util.GetConfig import GetConfig
22+
1923

20-
sys.path.append('../')
2124

2225
from Manager.ProxyManager import ProxyManager
2326

@@ -26,9 +29,10 @@
2629

2730
api_list = {
2831
'get': u'get an usable proxy',
29-
'refresh': u'refresh proxy pool',
32+
# 'refresh': u'refresh proxy pool',
3033
'get_all': u'get all proxy from proxy pool',
3134
'delete?proxy=127.0.0.1:8080': u'delete an unable proxy',
35+
'get_status': u'proxy statistics'
3236
}
3337

3438

@@ -54,7 +58,7 @@ def refresh():
5458
@app.route('/get_all/')
5559
def getAll():
5660
proxies = ProxyManager().getAll()
57-
return jsonify(list(proxies))
61+
return jsonify(proxies)
5862

5963

6064
@app.route('/delete/', methods=['GET'])
@@ -65,13 +69,14 @@ def delete():
6569

6670

6771
@app.route('/get_status/')
68-
def get_status():
69-
status = ProxyManager().get_status()
72+
def getStatus():
73+
status = ProxyManager().getNumber()
7074
return jsonify(status)
7175

7276

7377
def run():
74-
app.run(host='0.0.0.0', port=5000)
78+
config = GetConfig()
79+
app.run(host=config.host_ip, port=config.host_port)
7580

7681
if __name__ == '__main__':
7782
run()

‎Config.ini

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[DB]
22
;Configure the database information
3-
;type: SSDB/REDIS
3+
;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB
44
type = SSDB
55
host = localhost
66
port = 8888
@@ -13,3 +13,8 @@ freeProxySecond = 1
1313
freeProxyThird = 1
1414
freeProxyFourth = 1
1515
freeProxyFifth = 1
16+
17+
[HOST]
18+
; API接口配置 http://127.0.0.1:5051
19+
ip = 0.0.0.0
20+
port = 5010

‎DB/DbClient.py

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# !/usr/bin/env python
33
"""
44
-------------------------------------------------
5-
File Name： DbClient.py
5+
File Name： DbClient.py
66
Description : DB工厂类
77
Author : JHao
88
date： 2016/12/2
@@ -31,13 +31,16 @@ class DbClient(object):
3131
useful_proxy_queue： 存放检验后的代理；
3232
3333
抽象方法定义：
34-
get: 随机返回一个代理；
35-
put: 放回一个代理；
36-
getvalue: 返回代理属性（一个计数器）；
37-
inckey: 修改代理属性计数器的值;
38-
delete: 删除指定代理；
39-
getAll: 返回所有代理；
40-
changeTable: 切换 table or collection or hash;
34+
get(proxy): 返回proxy的信息；
35+
put(proxy): 存入一个代理；
36+
pop(): 弹出一个代理
37+
exists(proxy)： 判断代理是否存在
38+
getNumber(raw_proxy): 返回代理总数（一个计数器）；
39+
update(proxy, num): 修改代理属性计数器的值;
40+
delete(proxy): 删除指定代理；
41+
getAll(): 返回所有代理；
42+
changeTable(name): 切换 table or collection or hash;
43+
4144
4245
所有方法需要相应类去具体实现：
4346
SSDB：SsdbClient.py
@@ -65,39 +68,41 @@ def __initDbClient(self):
6568
__type = "SsdbClient"
6669
elif "REDIS" == self.config.db_type:
6770
__type = "RedisClient"
71+
elif "MONGODB" == self.config.db_type:
72+
__type = "MongodbClient"
6873
else:
6974
pass
7075
assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type)
7176
self.client = getattr(__import__(__type), __type)(name=self.config.db_name,
7277
host=self.config.db_host,
7378
port=self.config.db_port)
7479

75-
def get(self, **kwargs):
76-
return self.client.get(**kwargs)
80+
def get(self, key, **kwargs):
81+
return self.client.get(key, **kwargs)
7782

7883
def put(self, key, **kwargs):
7984
return self.client.put(key, **kwargs)
8085

81-
def getvalue(self, key, **kwargs):
82-
return self.client.getvalue(key, **kwargs)
83-
84-
def pop(self, **kwargs):
85-
return self.client.pop(**kwargs)
86-
87-
def inckey(self, key, value, **kwargs):
88-
return self.client.inckey(key, value, **kwargs)
86+
def update(self, key, value, **kwargs):
87+
return self.client.update(key, value, **kwargs)
8988

9089
def delete(self, key, **kwargs):
9190
return self.client.delete(key, **kwargs)
9291

92+
def exists(self, key, **kwargs):
93+
return self.client.exists(key, **kwargs)
94+
95+
def pop(self, **kwargs):
96+
return self.client.pop(**kwargs)
97+
9398
def getAll(self):
9499
return self.client.getAll()
95100

96101
def changeTable(self, name):
97102
self.client.changeTable(name)
98103

99-
def get_status(self):
100-
return self.client.get_status()
104+
def getNumber(self):
105+
return self.client.getNumber()
101106

102107

103108
if __name__ == "__main__":

‎DB/MongodbClient.py

Lines changed: 54 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,74 @@
11
# coding: utf-8
2-
3-
__author__ = 'Maps'
2+
"""
3+
-------------------------------------------------
4+
File Name： MongodbClient.py
5+
Description : 封装mongodb操作
6+
Author : JHao netAir
7+
date： 2017/3/3
8+
-------------------------------------------------
9+
Change Activity:
10+
2017/3/3:
11+
2017/9/26:完成对mongodb的支持
12+
-------------------------------------------------
13+
"""
14+
__author__ = 'Maps netAir'
415

516
from pymongo import MongoClient
6-
import random
7-
import json
817

918

1019
class MongodbClient(object):
20+
def __init__(self, name, host, port):
21+
self.name = name
22+
self.client = MongoClient(host, port)
23+
self.db = self.client.proxy
1124

12-
def __init__(self, name, host, port):
13-
self.name = name
14-
self.client = MongoClient(host, port)
15-
self.db = self.client.proxy
16-
17-
18-
def changeTable(self, name):
19-
self.name = name
20-
25+
def changeTable(self, name):
26+
self.name = name
2127

22-
def get(self):
23-
proxy = self.getAll()
24-
return random.choice(proxy) if proxy else None
28+
def get(self, proxy):
29+
data = self.db[self.name].find_one({'proxy': proxy})
30+
return data['num'] if data != None else None
2531

32+
def put(self, proxy, num=1):
33+
if self.db[self.name].find_one({'proxy': proxy}):
34+
return None
35+
else:
36+
self.db[self.name].insert({'proxy': proxy, 'num': num})
2637

27-
def put(self, value):
28-
if self.db[self.name].find_one({'proxy': value}):
29-
return None
30-
else:
31-
self.db[self.name].insert({'proxy': value})
38+
def pop(self):
39+
data = list(self.db[self.name].aggregate([{'$sample': {'size': 1}}]))
40+
if data:
41+
data = data[0]
42+
value = data['proxy']
43+
self.delete(value)
44+
return {'proxy': value, 'value': data['num']}
45+
return None
3246

47+
def delete(self, value):
48+
self.db[self.name].remove({'proxy': value})
3349

34-
def pop(self):
35-
value = self.get()
36-
if value:
37-
self.delete(value)
38-
return value
50+
def getAll(self):
51+
return {p['proxy']: p['num'] for p in self.db[self.name].find()}
3952

53+
def clean(self):
54+
self.client.drop_database('proxy')
4055

41-
def delete(self, value):
42-
self.db[self.name].remove({'proxy': value})
56+
def delete_all(self):
57+
self.db[self.name].remove()
4358

59+
def update(self, key, value):
60+
self.db[self.name].update({'proxy': key}, {'$inc': {'num': value}})
4461

45-
def getAll(self):
46-
return [p['proxy'] for p in self.db[self.name].find()]
62+
def exists(self, key):
63+
return True if self.db[self.name].find_one({'proxy': key}) != None else False
4764

48-
49-
def clean(self):
50-
self.client.drop_database('proxy')
51-
52-
53-
def delete_all(self):
54-
self.db[self.name].remove()
65+
def getNumber(self):
66+
return self.db[self.name].count()
5567

5668

5769
if __name__ == "__main__":
58-
db = MongodbClient('first', 'localhost', 27017)
59-
db.put('127.0.0.1:1')
60-
db2 = MongodbClient('second', 'localhost', 27017)
61-
db2.put('127.0.0.1:2')
62-
db.clean()
63-
70+
db = MongodbClient('first', 'localhost', 27017)
71+
# db.put('127.0.0.1:1')
72+
# db2 = MongodbClient('second', 'localhost', 27017)
73+
# db2.put('127.0.0.1:2')
74+
print(db.pop())

‎DB/RedisClient.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,17 @@
66
2017/4/17 修改pop
77
'''
88

9+
# ############################
10+
# 已弃用，
11+
# SsdbClient.py 支持redis
12+
##############################
13+
914
import json
1015
import random
1116
import redis
1217
import sys
1318

19+
1420
class RedisClient(object):
1521
"""
1622
Reids client
@@ -41,7 +47,7 @@ def get(self):
4147
return rkey.decode('utf-8')
4248
else:
4349
return rkey
44-
# return self.__conn.srandmember(name=self.name)
50+
# return self.__conn.srandmember(name=self.name)
4551

4652
def put(self, key):
4753
"""
@@ -87,7 +93,7 @@ def getAll(self):
8793
return [key.decode('utf-8') for key in self.__conn.hgetall(self.name).keys()]
8894
else:
8995
return self.__conn.hgetall(self.name).keys()
90-
# return self.__conn.smembers(self.name)
96+
# return self.__conn.smembers(self.name)
9197

9298
def get_status(self):
9399
return self.__conn.hlen(self.name)

0 commit comments

Comments
 (0)