代码一:
#!/usr/bin/python # -*- coding: utf-8 -*- #encoding=utf-8 import threading import Queue import sys import urllib2 import re import MySQLdb # # 数据库变量设置 # DB_HOST = '127.0.0.1' DB_USER = "XXXX" DB_PASSWD = "XXXXXXXX" DB_NAME = "xxxx" # # 变量设置 # THREAD_LIMIT = 3 jobs = Queue.Queue(5) singlelock = threading.Lock() info = Queue.Queue() def workerbee(inputlist): for x in xrange(THREAD_LIMIT): print 'Thead {0} started.'.format(x) t = spider() t.start() for i in inputlist: try: jobs.put(i, block=True, timeout=5) except: singlelock.acquire() print "The queue is full !" singlelock.release() # Wait for the threads to finish singlelock.acquire() # Acquire the lock so we can print print "Waiting for threads to finish." singlelock.release() # Release the lock jobs.join() # This command waits for all threads to finish. # while not jobs.empty(): # print jobs.get() def getTitle(url,time=10): response = urllib2.urlopen(url,timeout=time) html = response.read() response.close() reg = r'<title>(.*"htmlcode">#!/usr/bin/python # -*- coding: utf-8 -*- #encoding=utf-8 #Filename:robot.py import threading,Queue,sys,urllib2,re # # 变量设置 # THREAD_LIMIT = 3 #设置线程数 jobs = Queue.Queue(5) #设置队列长度 singlelock = threading.Lock() #设置一个线程锁,避免重复调用 urls = ['http://games.sina.com.cn/w/n/2013-04-28/1634703505.shtml','http://games.sina.com.cn/w/n/2013-04-28/1246703487.shtml','http://games.sina.com.cn/w/n/2013-04-28/1028703471.shtml','http://games.sina.com.cn/w/n/2013-04-27/1015703426.shtml','http://games.sina.com.cn/w/n/2013-04-26/1554703373.shtml','http://games.sina.com.cn/w/n/2013-04-26/1512703346.shtml','http://games.sina.com.cn/w/n/2013-04-26/1453703334.shtml','http://games.sina.com.cn/w/n/2013-04-26/1451703333.shtml','http://games.sina.com.cn/w/n/2013-04-26/1445703329.shtml','http://games.sina.com.cn/w/n/2013-04-26/1434703322.shtml','http://games.sina.com.cn/w/n/2013-04-26/1433703321.shtml','http://games.sina.com.cn/w/n/2013-04-26/1433703320.shtml','http://games.sina.com.cn/w/n/2013-04-26/1429703318.shtml','http://games.sina.com.cn/w/n/2013-04-26/1429703317.shtml','http://games.sina.com.cn/w/n/2013-04-26/1409703297.shtml','http://games.sina.com.cn/w/n/2013-04-26/1406703296.shtml','http://games.sina.com.cn/w/n/2013-04-26/1402703292.shtml','http://games.sina.com.cn/w/n/2013-04-26/1353703286.shtml','http://games.sina.com.cn/w/n/2013-04-26/1348703284.shtml','http://games.sina.com.cn/w/n/2013-04-26/1327703275.shtml','http://games.sina.com.cn/w/n/2013-04-26/1239703265.shtml','http://games.sina.com.cn/w/n/2013-04-26/1238703264.shtml','http://games.sina.com.cn/w/n/2013-04-26/1231703262.shtml','http://games.sina.com.cn/w/n/2013-04-26/1229703261.shtml','http://games.sina.com.cn/w/n/2013-04-26/1228703260.shtml','http://games.sina.com.cn/w/n/2013-04-26/1223703259.shtml','http://games.sina.com.cn/w/n/2013-04-26/1218703258.shtml','http://games.sina.com.cn/w/n/2013-04-26/1202703254.shtml','http://games.sina.com.cn/w/n/2013-04-26/1159703251.shtml','http://games.sina.com.cn/w/n/2013-04-26/1139703233.shtml'] def workerbee(inputlist): for x in xrange(THREAD_LIMIT): print 'Thead {0} started.'.format(x) t = spider() t.start() for i in inputlist: try: jobs.put(i, block=True, timeout=5) except: singlelock.acquire() print "The queue is full !" singlelock.release() # Wait for the threads to finish singlelock.acquire() # Acquire the lock so we can print print "Waiting for threads to finish." singlelock.release() # Release the lock jobs.join() # This command waits for all threads to finish. # while not jobs.empty(): # print jobs.get() def getTitle(url,time=10): response = urllib2.urlopen(url,timeout=time) html = response.read() response.close() reg = r'<title>(.*?)</title>' title = re.compile(reg).findall(html) title = title[0].decode('gb2312','replace').encode('utf-8') return title class spider(threading.Thread): def run(self): while 1: try: job = jobs.get(True,1) singlelock.acquire() title = getTitle(job) print 'This {0} is {1}'.format(job,title) singlelock.release() jobs.task_done() except: break; if __name__ == '__main__': workerbee(urls)
标签:
python,多线程采集
免责声明:本站文章均来自网站采集或用户投稿,网站不提供任何软件下载或自行开发的软件!
如有用户或公司发现本站内容信息存在侵权行为,请邮件告知! 858582#qq.com
白云城资源网 Copyright www.dyhadc.com
暂无“python实现多线程采集的2个代码例子”评论...
P70系列延期,华为新旗舰将在下月发布
3月20日消息,近期博主@数码闲聊站 透露,原定三月份发布的华为新旗舰P70系列延期发布,预计4月份上市。
而博主@定焦数码 爆料,华为的P70系列在定位上已经超过了Mate60,成为了重要的旗舰系列之一。它肩负着重返影像领域顶尖的使命。那么这次P70会带来哪些令人惊艳的创新呢?
根据目前爆料的消息来看,华为P70系列将推出三个版本,其中P70和P70 Pro采用了三角形的摄像头模组设计,而P70 Art则采用了与上一代P60 Art相似的不规则形状设计。这样的外观是否好看见仁见智,但辨识度绝对拉满。
更新日志
2024年12月26日
2024年12月26日
- 小骆驼-《草原狼2(蓝光CD)》[原抓WAV+CUE]
- 群星《欢迎来到我身边 电影原声专辑》[320K/MP3][105.02MB]
- 群星《欢迎来到我身边 电影原声专辑》[FLAC/分轨][480.9MB]
- 雷婷《梦里蓝天HQⅡ》 2023头版限量编号低速原抓[WAV+CUE][463M]
- 群星《2024好听新歌42》AI调整音效【WAV分轨】
- 王思雨-《思念陪着鸿雁飞》WAV
- 王思雨《喜马拉雅HQ》头版限量编号[WAV+CUE]
- 李健《无时无刻》[WAV+CUE][590M]
- 陈奕迅《酝酿》[WAV分轨][502M]
- 卓依婷《化蝶》2CD[WAV+CUE][1.1G]
- 群星《吉他王(黑胶CD)》[WAV+CUE]
- 齐秦《穿乐(穿越)》[WAV+CUE]
- 发烧珍品《数位CD音响测试-动向效果(九)》【WAV+CUE】
- 邝美云《邝美云精装歌集》[DSF][1.6G]
- 吕方《爱一回伤一回》[WAV+CUE][454M]