Skip to content

Commit 87b60b7

Browse files
rieuserieuse
committed
add spider
1 parent 70bb7d8 commit 87b60b7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

113 files changed

+258662
-3
lines changed

‎.idea/workspace.xml

Lines changed: 114 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎guardian/README.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
进程守护思路：
2+
1. 通过判断cpu 和内存占用率 然后决定是否要开始kill 程序
3+
2. kill 程序 需要获取到pid pid可通过 指定查找程序 name & 程序所在目录
4+
5+
6+
下面需要做的测试： 使用 taskkill mongodb 是否影响数据库

‎guardian/demo1.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import subprocess
2+
import os
3+
4+
# ["tasklist","|","python"]
5+
# res = subprocess.Popen(["tasklist","|","python"],stdout=subprocess.PIPE)
6+
res = subprocess.Popen("tasklist | grep python", stdout=subprocess.PIPE, shell=True)
7+
python_process = res.stdout.readlines()
8+
counts = len(python_process)
9+
10+
print(python_process, counts)
11+
# if counts < 4:
12+
# os.system('python /Users/mac/Desv7ktop/3-req.py')

‎guardian/demo2.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import os
2+
import signal
3+
import subprocess
4+
import psutil
5+
6+
pwd = os.getcwd()
7+
all_pid = psutil.pids()
8+
9+
10+
def get_info(pid):
11+
p = psutil.Process(i)
12+
print(p.name()) # 进程名
13+
print(p.exe()) # 进程的bin路径
14+
print(p.cwd()) # 进程的工作目录绝对路径
15+
print(p.create_time()) # 进程创建时间
16+
print(p.memory_percent()) # 进程内存利用率
17+
print(p.cpu_times()) # 进程的cpu时间信息,包括user,system两个cpu信息
18+
print(p.memory_percent()) # 进程内存利用率
19+
print(p.memory_info()) # 进程内存rss,vms信息
20+
print(p.num_threads()) # 进程开启的线程数
21+
22+
23+
if __name__ == '__main__':
24+
for pid in all_pid:
25+
p = psutil.Process(pid)
26+
if p.name() == 'python.exe':
27+
pid_pwd = p.cwd()
28+
if pwd == pid_pwd:
29+
print(pid)
30+
os.popen('taskkill.exe /pid:' + str(pid) + ' -f')
31+
# print(psutil.virtual_memory().percent)
32+
# psutil.pid_exists(pid)

‎guardian/guardian.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import multiprocessing
2+
import os
3+
import subprocess
4+
import psutil
5+
import time
6+
7+
pwd = os.getcwd()
8+
all_pid = psutil.pids()
9+
10+
11+
def judge_spider():
12+
for pid in all_pid:
13+
p = psutil.Process(pid)
14+
if p.name() == 'python.exe':
15+
pid_pwd = p.cwd()
16+
if pwd == pid_pwd:
17+
print('pid: ' + str(pid))
18+
memory_percent = psutil.virtual_memory().percent
19+
pid_memory_percent = p.memory_percent()
20+
print('memory_percent; ' + str(memory_percent), 'pid_memory_percent: ' + str(pid_memory_percent))
21+
# if memory_percent > 30 and pid_memory_percent > 0.17:
22+
# kill_python(pid)
23+
# kill_mongo()
24+
# time.sleep(5)
25+
# print('restart')
26+
# start_mongo()
27+
# start_spider()
28+
# judge_spider()
29+
30+
31+
def start_spider():
32+
os.system('python spider.py')
33+
print(os.getpid())
34+
35+
36+
def start_mongo():
37+
os.system('mongod --dbpath=F:\data\db')
38+
print('成功开启 mongodb')
39+
40+
41+
def kill_python(pid):
42+
os.popen('taskkill.exe /pid:' + str(pid) + ' -f')
43+
44+
45+
def kill_mongo():
46+
for pid in all_pid:
47+
p = psutil.Process(pid)
48+
if p.name() == 'mongod.exe':
49+
os.popen('taskkill.exe /pid:' + str(pid) + ' -f')
50+
51+
52+
if __name__ == '__main__':
53+
os.system('mongod --dbpath=F:\data\db')
54+
# p1 = multiprocessing.Process(target=start_mongo, args=())
55+
# p1.start()
56+
# time.sleep(3)
57+
print('成功开启 mongodb')
58+
# p2 = multiprocessing.Process(target=start_spider, args=())
59+
# p2.start()
60+
os.system('python spider.py')
61+
print(os.getpid())
62+
# print(p2.pid)
63+
for pid in all_pid:
64+
p = psutil.Process(pid)
65+
if p.name() == 'python.exe':
66+
print(p.name())
67+
# judge_spider()

‎guardian/guardian2.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# encoding=utf-8
2+
import psutil
3+
import time
4+
import sys
5+
6+
"""
7+
这个脚本用于监控某个进程的CPU和内存使用情况，将信息记录到文件中
8+
"""
9+
10+
if __name__ == '__main__':
11+
"""参数1:进程PID 参数2:保存的文件名(可选) 参数3:时间间隔(可选)
12+
"""
13+
# pid = sys.argv[1]
14+
pid = '4140'
15+
file_name = sys.argv[2] if len(sys.argv) > 2 else 'Test.txt'
16+
interval = float(sys.argv[3]) if len(sys.argv) > 3 else 1800
17+
p = psutil.Process(int(pid))
18+
ss = 'time: %s\tcpu percent: %f\tmemory usage:%d\n' % (
19+
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())),
20+
p.cpu_percent(interval=1.0),
21+
p.memory_info()[0])
22+
print(ss)
23+
# try:
24+
# while True:
25+
# with open(file_name, 'a') as f:
26+
# f.write('time: %s\tcpu percent: %f\tmemory usage:%d\n' % (
27+
# time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())),
28+
# p.get_cpu_percent(interval=1.0),
29+
# p.get_memory_info()[0]))
30+
# # 每一段时间记录一次进程的CPU和内存使用信息
31+
# time.sleep(interval)
32+
# except KeyboardInterrupt:
33+
# # 中断退出
34+
# print('Exit!')

0 commit comments

Comments
 (0)