在Python中使用CasperJS获取JS渲染生成的HTML内容的教程

 import json 
 import sys 
 #import requests 
 #import requests.utils, pickle 
 from bs4 import BeautifulSoup 
 import os.path,os 
 import threading 
 #from multiprocessing import Process, Manager 
 from datetime import datetime 
 import traceback 
 import logging 
 import re,random 
 import subprocess 
 import shutil 
 import platform 
 
 
 
 
 output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt') 
 global_log = 'http_proxy' + datetime.now().strftime('%Y-%m-%d') + '.log' 
 if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')): 
 os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')) 
 global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log) 
 
 logging.basicConfig(level=logging.DEBUG,format='[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s',filename=global_log,filemode='a') 
 log = logging.getLogger(__name__) 
 #manager = Manager() 
 #PROXY_LIST = manager.list() 
 mutex = threading.Lock() 
 PROXY_LIST = [] 
 
 
 def isWindows(): 
 if "Windows" in str(platform.uname()): 
 return True 
 else: 
 return False 
 
 
 def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue): 
 soup = BeautifulSoup(pageContent) 
 return soup.find_all(tagName, { attrName : re.compile(attrRegValue) }) 
 
 
 def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue): 
 if os.path.isfile(filename): 
 f = open(filename,'r') 
 soup = BeautifulSoup(f) 
 f.close() 
 return soup.find_all(tagName, { attrName : re.compile(attrRegValue) }) 
 else: 
 return None 
 
 
 class Site1Thread(threading.Thread): 
 def __init__(self,outputFilePath): 
 threading.Thread.__init__(self) 
 self.outputFilePath = outputFilePath 
 self.fileName = str(random.randint(100,1000)) + ".html" 
 self.setName('Site1Thread') 
 
 def run(self): 
 site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js') 
 site2_file = os.path.join(self.outputFilePath,'site.js') 
 if not os.path.isfile(site2_file) and os.path.isfile(site1_file): 
 shutil.copy(site1_file,site2_file) 
 #proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
 if isWindows(): 
 proc = subprocess.Popen(["cmd","/c", "%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
 else: 
 proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) 
 out=proc.communicate()[0] 
 htmlFileName = '' 
 #因为