python实现批量下载新浪博客的方法

# coding=utf-8 
import urllib2
import sys, os
import re
import string
from BeautifulSoup import BeautifulSoup
def encode(s):
 return s.decode('utf-8').encode(sys.stdout.encoding, 'ignore')
def getHTML(url):
 #proxy_handler = urllib2.ProxyHandler({'http':'http://211.138.124.211:80'})
 #opener = urllib2.build_opener(proxy_handler)
 #urllib2.install_opener(opener)
 req = urllib2.Request(url)
 response = urllib2.urlopen(req, timeout=15)
 return BeautifulSoup(response, convertEntities=BeautifulSoup.HTML_ENTITIES)
def visible(element):
 '''抓取可见的文本元素'''
 if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
 return False
 elif re.match('', str(element)):
 return False
 elif element == u'xa0':
 return False
 return True
def delReturn(element):
 '''删除元素内的换行'''
 return re.sub('(?"|xa0]', '', filename)
def writeToFile(text, filename, dirname):
 if not os.path.exists(dirname):
 os.makedirs(dirname)
 print encode('保存到目录'), dirname
 filename = validFilename(filename)
 print encode('保存文章'), filename
 path = os.path.join(dirname, filename)
 if not os.path.exists(path):
 f = open(path, 'w')
 f.write(text)
 f.close()
 else:
 print filename, encode('已经存在')
def formatContent(url, title=''):
 '''格式化文章内容'''
 page = getHTML(url)
 content = page.find('div', {'class':'articalContent'})
 art_id = re.search('blog_(w+).html', url).group(1)
 blog_name = page.find('span', id='blognamespan').string
 if title == '':
 title = page.find('h2', id=re.compile('^t_')).string
 temp_data = filter(visible, content.findAll(text=True)) # 去掉不可见元素
 temp_data = ''.join(map(delReturn, temp_data)) # 删除元素内的换行符
 temp_data = temp_data.strip() # 删除文章首尾的空行
 temp_data = re.sub('
{2,}', '

', temp_data) # 删除文章内过多的空行
 #