114python之家

您现在的位置是:首页 > python > 正文

python

python进阶 多线程编程 —— threading和queue库实现多线程编程

admin2021-02-06python386
【114python之家】python进阶多线程编程——threading和queue库实现多线程编程)摘要多线程实现逻辑封装模型参数选择实例摘要本文主要介绍了利用python的threading和q

python进阶多线程编程——threading和queue库实现多线程编程)

  • 摘要
  • 多线程实现逻辑封装
  • 模型参数选择实例

摘要

本文主要介绍了利用python的threading和queue库实现多线程编程,并封装为一个类,方便读者嵌入自己的业务逻辑。最后以机器学习的一个超参数选择为例进行演示。

多线程实现逻辑封装

实例化该类后,在.object_func函数中加入自己的业务逻辑,再调用.run方法即可。

#-*-coding:utf-8-*-
#@Time:2021/2/414:36
#@Author:CyrusMayWJ
#@FileName:run.py
#@Software:PyCharm
#@Blog:https://blog.csdn.net/Cyrus_May
importqueue
importthreading

classCyrusThread(object):
def__init__(self,num_thread=10,logger=None):
"""

:paramnum_thread:线程数
:paramlogger:日志对象
"""
self.num_thread=num_thread
self.logger=logger

defobject_func(self,args_queue,max_q):
while1:
try:
arg=args_queue.get_nowait()
step=args_queue.qsize()
self.logger.info("progress:{}{}".format(max_q,step))
except:
self.logger.info("nomoreargforargs_queue!")
break


"""
此处加入自己的业务逻辑代码
"""


defrun(self,args):
args_queue=queue.Queue()
forvalueinargs:
args_queue.put(value)
threads=[]
foriinrange(self.num_thread):
threads.append(threading.Thread(target=self.object_func,args=args_queue))
fortinthreads:
t.start()
fortinthreads:
t.join()

模型参数选择实例

#-*-coding:utf-8-*-
#@Time:2021/2/414:36
#@Author:CyrusMayWJ
#@FileName:run.py
#@Software:PyCharm
#@Blog:https://blog.csdn.net/Cyrus_May
importqueue
importthreading
importnumpyasnp
fromsklearn.datasetsimportload_boston
fromsklearn.svmimportSVR
importlogging
importsys


classCyrusThread(object):
def__init__(self,num_thread=10,logger=None):
"""

:paramnum_thread:线程数
:paramlogger:日志对象
"""
self.num_thread=num_thread
self.logger=logger

defobject_func(self,args_queue,max_q):
while1:
try:
arg=args_queue.get_nowait()
step=args_queue.qsize()
self.logger.info("progress:{}{}".format(max_q,max_q-step))
except:
self.logger.info("nomoreargforargs_queue!")
break
#业务代码
C,epsilon,gamma=arg[0],arg[1],arg[2]
svr_model=SVR(C=C,epsilon=epsilon,gamma=gamma)
x,y=load_boston()["data"],load_boston()["target"]
svr_model.fit(x,y)
self.logger.info("score:{}".format(svr_model.score(x,y)))


defrun(self,args):
args_queue=queue.Queue()
max_q=0
forvalueinargs:
args_queue.put(value)
max_q+=1
threads=[]
foriinrange(self.num_thread):
threads.append(threading.Thread(target=self.object_func,args=(args_queue,max_q)))
fortinthreads:
t.start()
fortinthreads:
t.join()

#创建日志对象
logger=logging.getLogger()
logger.setLevel(logging.INFO)
screen_handler=logging.StreamHandler(sys.stdout)
screen_handler.setLevel(logging.INFO)
formatter=logging.Formatter('%(asctime)s-%(module)s.%(funcName)s:%(lineno)d-%(levelname)s-%(message)s')
screen_handler.setFormatter(formatter)
logger.addHandler(screen_handler)

#创建需要调整参数的集合
args=[]
forCin[iforiinnp.arange(0.01,1,0.01)]:
forepsilonin[iforiinnp.arange(0.001,1,0.01)]+[iforiinrange(1,10,1)]:
forgammain[iforiinnp.arange(0.001,1,0.01)]+[iforiinrange(1,10,1)]:
args.append([C,epsilon,gamma])

#创建多线程工具
threading_tool=CyrusThread(num_thread=20,logger=logger)
threading_tool.run(args)

运行结果

2021-02-0420:52:22,824-run.object_func:31-INFO-progress:11762191
2021-02-0420:52:22,824-run.object_func:31-INFO-progress:11762192
2021-02-0420:52:22,826-run.object_func:31-INFO-progress:11762193
2021-02-0420:52:22,833-run.object_func:31-INFO-progress:11762194
2021-02-0420:52:22,837-run.object_func:31-INFO-progress:11762195
2021-02-0420:52:22,838-run.object_func:31-INFO-progress:11762196
2021-02-0420:52:22,841-run.object_func:31-INFO-progress:11762197
2021-02-0420:52:22,862-run.object_func:31-INFO-progress:11762198
2021-02-0420:52:22,873-run.object_func:31-INFO-progress:11762199
2021-02-0420:52:22,884-run.object_func:31-INFO-progress:117621910
2021-02-0420:52:22,885-run.object_func:31-INFO-progress:117621911
2021-02-0420:52:22,897-run.object_func:31-INFO-progress:117621912
2021-02-0420:52:22,900-run.object_func:31-INFO-progress:117621913
2021-02-0420:52:22,904-run.object_func:31-INFO-progress:117621914
2021-02-0420:52:22,912-run.object_func:31-INFO-progress:117621915
2021-02-0420:52:22,920-run.object_func:31-INFO-progress:117621916
2021-02-0420:52:22,920-run.object_func:39-INFO-score:-0.01674283914287855
2021-02-0420:52:22,929-run.object_func:31-INFO-progress:117621917
2021-02-0420:52:22,932-run.object_func:39-INFO-score:-0.007992354170952565
2021-02-0420:52:22,932-run.object_func:31-INFO-progress:117621918
2021-02-0420:52:22,945-run.object_func:31-INFO-progress:117621919
2021-02-0420:52:22,954-run.object_func:31-INFO-progress:117621920
2021-02-0420:52:22,978-run.object_func:31-INFO-progress:117621921
2021-02-0420:52:22,984-run.object_func:39-INFO-score:-0.018769934807246536
2021-02-0420:52:22,985-run.object_func:31-INFO-progress:117621922

byCyrusMay20210204

在某个清晨
回望我一生
活得虽认真
却微小如尘
——————五月天(一半人生)——————

扫描关注公众号,第一时间获取网站更新动态

转载请说明来源于"114python之家"

本文地址:http://www.114python.com/post/9604.html

发表评论

评论列表

  • 这篇文章还没有收到评论,赶紧来抢沙发吧~