进一步优化
- 使用phantomjs替代firefox,实现无浏览器界面,加快速度
- 去除上版对于浏览器布局部分代码
优化后的结果
基本上可以在5分钟内搞定上海的股票数据下载比过去确实有大进步
#coding=utf-8
from selenium import webdriver
import time
import os
import re
import sys
import threading
import Queue
from selenium.common.exceptions import NoSuchElementException
def myinit():
reload(sys)
sys.setdefaultencoding(\'utf8\')
########################
#生成多个浏览器实例
def makebr(number):
brs=[]
for i in range(number):
#br=webdriver.Firefox()
br=webdriver.PhantomJS(\'phantomjs\')
brs.append(br)
return brs
#关闭所有浏览器
def closebr(brs):
for i in brs:
i.quit()
#######################
#获得指定股票相关数据
def getinfo(mydriver,gourl):
title=\'//*[@id=\"name\"]\'
code=\'//*[@id=\"code\"]\'
hexinshujuxpath=\"/html/body/div[14]/div[1]/div[4]/div[1]\"
restr=\":\".decode(\'utf8\')
myre=re.compile(restr,re.I|re.M|re.S)
linetext=\"\"
errorinfo=\"\"
mydriver.get(gourl)
try:
gupiaoming=mydriver.find_element_by_xpath(title).text
gupiaocode=mydriver.find_element_by_xpath(code).text
hexinshuju=mydriver.find_element_by_class_name(\'pad5\')
shujuhang=hexinshuju.find_elements_by_tag_name(\'tr\')
for i in range(len(shujuhang)-2):
shujulie=shujuhang[i].find_elements_by_tag_name(\'td\')
tmpshuju=myre.split(shujulie[0].text)
linetext=linetext+\"~\"+tmpshuju[1]
shuju=myre.split(shujuhang[8].text)
linetext=linetext+\"~\"+shuju[1]
tmpshuju=myre.split(shujuhang[9].text)
linetext=linetext+\"~\"+tmpshuju[1]
linetext=\"%s~%s%s\"%(gupiaoming,gupiaocode,linetext)
#print \"数据:\",linetext
except NoSuchElementException,e:
#print \"不是股票\"
pass
except Exception:
errorinfo= \"非预期错误\"+gourl
print errorinfo
finally:
return linetext,errorinfo
#获得所有股票链接
def geturls(br):
#通过link对象获得链接地址的text文本
def getlinkurl(linklist):
my=[]
for x in linklist:
my.append(x.get_attribute(\'href\'))
return my
sz=[]
sh=[]
br.get(\"http://quote.eastmoney.com/stocklist.html\")
shxpath=\"/html/body/div[9]/div[2]/div/ul[1]\"
szxpath=\"/html/body/div[9]/div[2]/div/ul[2]\"
shgupiao=br.find_element_by_xpath(shxpath)
szgupiao=br.find_element_by_xpath(szxpath)
shgupiaolist=shgupiao.find_elements_by_tag_name(\'a\')
szgupiaolist=szgupiao.find_elements_by_tag_name(\'a\')
sh=getlinkurl(shgupiaolist)
sz=getlinkurl(szgupiaolist)
return sh,sz
#多线程执行用的函数
def thread_getinfo(br,jobslink_queue,jieguo_queue,errorinfo_queue):
while True:
try:
#获得队列里的地址
url=jobslink_queue.get(False) #False =Don\'t wait
except Queue.Empty:
print \"完成退出\"
#br.quit()
return
#print url
if(url!=None):
(linetext,errorinfo)=getinfo(br,url)
if(linetext!=\"\"):
jieguo_queue.put(linetext)
if(errorinfo!=\"\"):
errorinfo_queue.put(errorinfo)
#######################
#多线程控制函数
def saveinfoabc(info_filename,error_filename,urllist):
jobslink=Queue.Queue(0)
jieguo=Queue.Queue(0)
errorsinfo=Queue.Queue(0)
for x in urllist[200:250]:#为测试方便这里只取了50,如果要全下载,取消[]就好
#for x in urllist:
jobslink.put(x)
#启动线程
for x in range(THREAD_NUM):
t=threading.Thread(target=thread_getinfo,args=(brs[x],jobslink,jieguo,errorsinfo))
t.start()
f=open(info_filename,\'w\')
e=open(error_filename,\'w\')
mycount=0
while (threading.activeCount()>1) or (not jobslink.empty()):
while jieguo.qsize()>0 or errorsinfo.qsize()>0:
if(jieguo.qsize()>0):
jieguotxt=jieguo.get()
f.write(jieguotxt+\"\\n\")
if(errorsinfo.qsize()>0):
error=errorsinfo.get()
e.write(error+\"\\n\")
mycount=mycount+1
if(mycount%100)==0:
print \"%d: 活动线程:%d,剩余连接数:%d,结果剩余条数:%d,错误剩余条数:%d\"%(mycount,threading.activeCount(),jobslink.qsize(),jieguo.qsize(),errorsinfo.qsize())
time.sleep(0.01)
e.close()
f.close()
print \"数据下载完成\"
######################
myinit()
#br=webdriver.Firefox()
br=webdriver.PhantomJS(\'phantomjs\')
print \"获得所有链接地址\"
(sh,sz)=geturls(br)
br.quit()
#info_filename=\'shinfo.txt\'
#error_filename=\'sherror.txt\'
THREAD_NUM=10
brs=makebr(THREAD_NUM)
saveinfoabc(\'shinfo_for_008.txt\',\'sherror_for_008.txt\',sh)
saveinfoabc(\'szinfo.txt\',\'szerror.txt\',sz)
closebr(brs)