selenium+phantomjs爬取微博数据

import time

import sys

#reload(sys)

#sys.setdefaultencoding('utf-8')

from weibo import APIClient

import json

# import webbrowser

import io

import re

import cookielib

from scrapy.selector import Selector

from scrapy.spiders import  CrawlSpider,Spider

from scrapy.http import Request,FormRequest

#selenium+phantomJS获取微博数据

from selenium import webdriver

#print "hello"

cap = webdriver.DesiredCapabilities.PHANTOMJS

cap["phantomjs.page.settings.resourceTimeout"] = 1000

cap["phantomjs.page.settings.loadImages"] = False

cap["phantomjs.page.settings.localToRemoteUrlAccessEnabled"] = True

driver = webdriver.PhantomJS(desired_capabilities=cap)

#driver = webdriver.PhantomJS()

print u'准备登陆Weibo.cn网站...'

driver.get("http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)")

# elem_user = driver.find_element_by_xpath("//div[@class='info_list username']/div[@class='input_wrap W_input_focus']")

elem_user = driver.find_element_by_xpath("//ul[@class='mainlogin']/li/div/div/span/cite/input[2]")

elem_user.send_keys('XXXXX') #用户名

elem_pwd  = driver.find_element_by_xpath("//ul[@class='mainlogin']/li[2]/div/div/span/cite/input")

elem_pwd.send_keys('XXXXX')  #密码

elem_sub = driver.find_element_by_xpath("//li/div[@class='ml_r']/input")

elem_sub.send_keys(u'登 录')  #密码

elem_sub.click()              #点击登陆

time.sleep(5)

try:

    driver.get("http://weibo.com/u/2865101843?from=feed&loc=nickname")

    page_source = driver.page_source

    response = page_source.encode('utf-8')

    yy=u"[\一-\龥|A-Za-z0-9_]"

xx=u"nick-name=\\"[\一-\龥|A-Za-z0-9_]+\\">"

    pattern = re.compile(xx)

    match = pattern.findall(page_source)

    for the_text in match:

        print the_text

    driver.quit()

finally:

    # 退出phantomjs进程，否则会累积越来越多的进程

    driver.quit()

    print '有错误，关闭phantomjs。'"""