文章摘要：其实这里casperjs与python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来，爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难, Java里面有 HtmlUnit, 而Python里，我们可以使用独立的跨平台的CasperJS。

    创建site.js(接口文件，输入:url，输出:html file)  

   //USAGE: E:\\toolkit\\n1k0-casperjs-e3a77d0\\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=\'temp.html\' 
     
    var fs = require(\'fs\'); 
    var casper = require(\'casper\').create({ 
     pageSettings: { 
     loadImages: false,     
     loadPlugins: false,    
     userAgent: \'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER\' 
    }, 
    logLevel: \"debug\",//日志等级 
    verbose: true  // 记录日志到控制台 
     }); 
    var url = casper.cli.raw.get(\'url\'); 
    var outputfile = casper.cli.raw.get(\'outputfile\'); 
    //请求页面 
    casper.start(url, function () { 
    fs.write(outputfile, this.getHTML(), \'w\'); 
    }); 
     
    casper.run(); 

    python 代码, checkout_proxy.py      

 import json 
    import sys 
    #import requests 
    #import requests.utils, pickle 
    from bs4 import BeautifulSoup 
    import os.path,os 
    import threading 
    #from multiprocessing import Process, Manager 
    from datetime import datetime 
    import traceback 
    import logging 
    import re,random 
    import subprocess 
    import shutil 
    import platform 
      
     
     
     
    output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),\'proxy.txt\') 
    global_log = \'http_proxy\' + datetime.now().strftime(\'%Y-%m-%d\') + \'.log\' 
    if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),\'logs\')): 
      os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),\'logs\')) 
    global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),\'logs\',global_log) 
     
    logging.basicConfig(level=logging.DEBUG,format=\'[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s\',filename=global_log,filemode=\'a\') 
    log = logging.getLogger(__name__)  
    #manager = Manager() 
    #PROXY_LIST = manager.list() 
    mutex = threading.Lock() 
    PROXY_LIST = [] 
     
     
    def isWindows(): 
      if \"Windows\" in str(platform.uname()): 
      return True 
      else: 
      return False 
     
     
    def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue): 
      soup = BeautifulSoup(pageContent)                                                 
      return soup.find_all(tagName, { attrName : re.compile(attrRegValue) }) 
     
     
    def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue): 
      if os.path.isfile(filename): 
      f = open(filename,\'r\')    
      soup = BeautifulSoup(f) 
      f.close() 
      return soup.find_all(tagName, { attrName : re.compile(attrRegValue) }) 
      else: 
      return None 
     
     
    class Site1Thread(threading.Thread): 
      def __init__(self,outputFilePath): 
        threading.Thread.__init__(self) 
      self.outputFilePath = outputFilePath 
      self.fileName = str(random.randint(100,1000)) + \".html\" 
      self.setName(\'Site1Thread\') 
      
      def run(self): 
      site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),\'site.js\') 
      site2_file = os.path.join(self.outputFilePath,\'site.js\') 
      if not os.path.isfile(site2_file) and os.path.isfile(site1_file): 
        shutil.copy(site1_file,site2_file) 
      #proc = subprocess.Popen([\"bash\",\"-c\", \"cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s\" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
      if isWindows(): 
        proc = subprocess.Popen([\"cmd\",\"/c\", \"%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s\" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
      else: 
        proc = subprocess.Popen([\"bash\",\"-c\", \"cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s\" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
      out=proc.communicate()[0] 
      htmlFileName = \'\' 
      #因为输出路径在windows不确定，所以这里加了所有可能的路径判断 
      if os.path.isfile(self.fileName): 
        htmlFileName = self.fileName 
      elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)): 
        htmlFileName = os.path.join(self.outputFilePath,self.fileName) 
      elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)): 
        htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)  
      if (not os.path.isfile(htmlFileName)): 
        print \'Failed to get html content from http://spys.ru/free-proxy-list/IE/\' 
        print out 
        sys.exit(3)  
      mutex.acquire() 
      PROXYList= getTagsByAttrsExt(\'font\',htmlFileName,\'class\',\'spy14$\') 
      for proxy in PROXYList: 
        tdContent = proxy.renderContents() 
        lineElems = re.split(\'[<>]\',tdContent) 
        if re.compile(r\'\\d+\').search(lineElems[-1]) and re.compile(\'(\\d+\\.\\d+\\.\\d+)\').search(lineElems[0]): 
        print lineElems[0],lineElems[-1] 
        PROXY_LIST.append(\"%s:%s\" % (lineElems[0],lineElems[-1])) 
      mutex.release() 
      try: 
        if os.path.isfile(htmlFileName): 
        os.remove(htmlFileName) 
      except: 
        pass 
     
    if __name__ == \'__main__\': 
      try: 
      if(len(sys.argv)) < 2: 
        print \"Usage:%s [casperjs path]\" % (sys.argv[0]) 
        sys.exit(1)  
      if not os.path.exists(sys.argv[1]): 
        print \"casperjs path: %s does not exist!\" % (sys.argv[1]) 
        sys.exit(2)  
      if os.path.isfile(output_file): 
        f = open(output_file) 
        lines = f.readlines() 
        f.close 
        for line in lines: 
        PROXY_LIST.append(line.strip()) 
      thread1 = Site1Thread(sys.argv[1]) 
      thread1.start() 
      thread1.join() 
       
      f = open(output_file,\'w\') 
      for proxy in set(PROXY_LIST): 
        f.write(proxy+\"\\n\") 
      f.close() 
      print \"Done!\" 
      except SystemExit: 
      pass 
      except: 
        errMsg = traceback.format_exc() 
        print errMsg 
        log.error(errMsg)