IT博客汇 | [原]Python多进程编程（一）：初探

[原]Python多进程编程（一）：初探

linwaterbin发表于 2014-04-13 22:21:08

1 基础例子

比较常用的做法是，创建一个进程时可以提供参数来告诉他要做什么。本例子里，输出的"worker"将打印5次，不过不清楚孰先孰后，因为每个进程都在竞争访问输出流。

1.1 输出顺序的不同

[root@localhost pydoc]# ./tmp.py
worker 0
worker 1
worker 2
worker 3
worker 4
[root@localhost pydoc]# ./tmp.py
worker 0
worker 1
worker 2
worker 4
worker 3

1.2 简单的代码实现

[root@localhost pydoc]# cat tmp.py
#!/usr/bin/env python
#coding=utf-8

from multiprocessing import Process

def worker(num):
  """test python multi process"""
  print 'worker',num
  return

def main():
  jobs = []
  for i in range(5):
    p = Process(target=worker,args=(i,))
    jobs.append(p)
    p.start()

if __name__=='__main__':
  main()

2 僵尸进程

要等待一个进程完成工作并退出，可以使用join()方法。在Linux上，当某个进程终结之后，需要被主进程调用wait，否则进程会成为僵尸进程。所以，有必要对每个Process对象调用join()方法，实际上等同于wait。

[root@localhost pydoc]# cat tmp.py 
#!/usr/bin/env python
#coding=utf-8

from multiprocessing import Process

def worker(num):
  """test python multi process"""
  print 'worker',num
  return

def main():
  jobs = []
  for i in range(5):
    p = Process(target=worker,args=(i,))
    jobs.append(p)
    p.start()
  ##to avoid defunct process,you should call join()
  for j in jobs:
    j.join()

if __name__=='__main__':
  main()

3 开源项目MTOP

MySQL MTOP是一个由Python+PHP开发的开源MySQL企业级监控系统，系统由Python实现多进程数据采集和监控，PHP实现web展示和管理。

有兴趣的同学可以参考它的官方网站 www.mtop.cc，这里主要介绍它是如何用Python来开发多进程数据采集和监控。

3.1 启动MTOP

[root@localhost mysqlmtop]# mtopctl status
mysql mtop is not run!
[root@localhost mysqlmtop]# mtopctl start
nohup: 把输出追加到"nohup.out"
mysql mtop start success!
[root@localhost mysqlmtop]# mtopctl status
mysql mtop is run!
[root@localhost mysqlmtop]# ps -ef | grep mtop
root      4463  4446  0 11:21 pts/2    00:00:00 vim mtopd.py
root      7119     1  0 14:50 pts/3    00:00:00 /bin/bash /usr/local/bin/mtopctl start
root      7121  7119  1 14:50 pts/3    00:00:00 python ./mtopd.py
root      7125  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7126  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7129  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7130  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7133  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7134  7121  0 14:50 pts/3    00:00:00 python ./mtopd.py
root      7186  6613  0 14:50 pts/3    00:00:00 grep mtop

从ps的输出我们也能清楚知道，pid为7121的父进程产生了6个子进程

3.2 MTOP多进程数据采集的主干代码

代码比较简单，看下基础例子基本也就明了。

[root@localhost mysqlmtop]# cat mtopd.py 
#!/bin/env python
#coding:utf-8
import os, sys, string, time, datetime, traceback;
from multiprocessing import Process;
import global_functions as func  

  


def job_run(script_name,times):
    while True:
        os.system("./"+script_name)
        time.sleep(int(times))



def main():
    print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
    monitor = func.get_option('monitor')
    monitor_status = func.get_option('monitor_status')
    monitor_replication = func.get_option('monitor_replication')
    monitor_process = func.get_option('monitor_process')
    alarm = func.get_option('alarm')
    frequency_monitor = func.get_option('frequency_monitor')
    frequency_alarm = func.get_option('frequency_alarm')
    kill_process = func.get_option('kill_process')

    joblist = []
    if monitor=="1":
        if monitor_status=="1":
            job = Process(target = job_run, args = ('check_mysql_status_ext.py',frequency_monitor))
            joblist.append(job)
            job.start()
            job = Process(target = job_run, args = ('check_mysql_status.py',frequency_monitor))
            joblist.append(job)
            job.start()
        if monitor_replication=="1":
            job = Process(target = job_run, args = ('check_mysql_replication.py',frequency_monitor))
            joblist.append(job)
            job.start()
        if monitor_process=="1":
            job = Process(target = job_run, args = ('check_mysql_process.py',4))
            joblist.append(job)
            job.start()
        if alarm=="1":
            job = Process(target = job_run, args = ('alarm_mysql.py',frequency_alarm))
            joblist.append(job)
            job.start()    
        if kill_process=="1":
            job = Process(target = job_run, args = ('admin_mysql_kill_process.py',3))
            joblist.append(job)
            job.start()

        for job in joblist:
            job.join();
    print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
    

  
if __name__ == '__main__':  
    main()

Good Luck！