python使用scrapy解析js示例
复制代码 代码如下:
from selenium import selenium
class MySpider(CrawlSpider):
name = \’cnbeta\’
allowed_domains = [\’cnbeta.com\’]
start_urls = [\’http://www.jb51.net\’]
rules = (
# Extract links matching \’category.php\’ (but not matching \’subsection.php\’)
# and follow links from them (since no callback means follow=True by default).
Rule(SgmlLinkExtractor(allow=(\’/articles/.*\\.htm\’, )),
callback=\’parse_page\’, follow=True),
# Extract links matching \’item.php\’ and parse them with the spider\’s method parse_item
)
def __init__(self):
CrawlSpider.__init__(self)
self.verificationErrors = []
self.selenium = selenium(\”localhost\”, 4444, \”*firefox\”, \”http://www.jb51.net\”)
self.selenium.start()
def __del__(self):
self.selenium.stop()
print self.verificationErrors
CrawlSpider.__del__(self)
def parse_page(self, response):
self.log(\’Hi, this is an item page! %s\’ % response.url)
sel = Selector(response)
from webproxy.items import WebproxyItem
sel = self.selenium
sel.open(response.url)
sel.wait_for_page_to_load(\”30000\”)
import time
time.sleep(2.5)