ExportCSDNBlog.py

#! encoding=utf-8

# Author : kesalin@gmail.com
# Blog : http://kesalin.github.io
# Date : 2014/10/18
# Description : Export CSND blog articles to Markdown files. 
# Version : 1.0.0.0
# Python Version: Python 2.7.3
#

import urllib2
import re
import os
import sys
import datetime
import time
import traceback
import codecs
from bs4 import BeautifulSoup

#===========================================================================
# set your CSDN username
__username__ = "kesalin"

# set output dir
__output__ = "C:/Code/Python"

enableLog = True

# for test
#__testArticleUrl__ = "http://blog.csdn.net/kesalin/article/details/5414998"

#===========================================================================

# 尝试获取资源次数
gRetryCount = 5
header = {"User-Agent": "Mozilla-Firefox5.0"}

def log(str):
 if enableLog:
 print str

 newFile = open('log.txt', 'a+')
 newFile.write(str + '\n')
 newFile.close()
 
def decodeHtmlSpecialCharacter(htmlStr):
 specChars = {" " : "", \
 " " : "", \
 " " : "", \
 "&lt;" : "<", \
 "&gt" : ">", \
 "&amp;" : "&", \
 "&quot;" : "\"", \
 "&copy;" : "®", \
 "&times;" : "×", \
 "&divide;" : "÷", \
 }
 for key in specChars.keys():
 htmlStr = htmlStr.replace(key, specChars[key])
 return htmlStr

def repalceInvalidCharInFilename(filename):
 specChars = {"\\" : "", \
 "/" : "", \
 ":" : "", \
 "*" : "", \
 "?" : "", \
 "\"" : "", \
 "<" : "小于", \
 ">" : "大于", \
 "|" : " and ", \
 "&" :" or ", \
 }
 for key in specChars.keys():
 filename = filename.replace(key, specChars[key])
 return filename

# process html content to markdown content
def htmlContent2String(contentStr):
 patternImg = re.compile(r'(<img.+?src=")(.+?)(".+ />)')
 patternHref = re.compile(r'(<a.+?href=")(.+?)(".+?>)(.+?)(</a>)')
 patternRemoveHtml = re.compile(r'</?[^>]+>')

 resultContent = patternImg.sub(r'![image_mark](\2)', contentStr)
 resultContent = patternHref.sub(r'[\4](\2)', resultContent)
 resultContent = re.sub(patternRemoveHtml, r'', resultContent)
 resultContent = decodeHtmlSpecialCharacter(resultContent)
 return resultContent

def exportToMarkdown(exportDir, postdate, categories, title, content):
 titleDate = postdate.strftime('%Y-%m-%d')
 contentDate = postdate.strftime('%Y-%m-%d %H:%M:%S %z')
 filename = titleDate + '-' + title
 filename = repalceInvalidCharInFilename(filename)
 filepath = exportDir + '/' + filename + '.markdown'
 log(" >> save as " + filename)

 newFile = open(unicode(filepath, "utf8"), 'w')
 newFile.write('---' + '\n')
 newFile.write('layout: post' + '\n')
 newFile.write('title: \"' + title + '\"\n')
 newFile.write('date: ' + contentDate + '\n')
 newFile.write('comments: true' + '\n')
 newFile.write('categories: [' + categories + ']' + '\n')
 newFile.write('tags: [' + categories + ']' + '\n')
 newFile.write('description: \"' + title + '\"\n')
 newFile.write('keywords: ' + categories + '\n') 
 newFile.write('---' + '\n\n')
 newFile.write(content)
 newFile.write('\n')
 newFile.close()

def download(url, output):
 # 下载文章，并保存为 markdown 格式
 log(" >> download: " + url)

 data = None
 title = ""
 categories = ""
 content = ""
 postDate = datetime.datetime.now()
 
 global gRetryCount
 count = 0
 while True:
 if count >= gRetryCount:
 break
 count = count + 1
 try:
 time.sleep(2.0) #访问太快会不响应
 request = urllib2.Request(url, None, header)
 response = urllib2.urlopen(request)
 data = response.read().decode('UTF-8')
 break
 except Exception,e:
 exstr = traceback.format_exc()
 log(" >> failed to download " + url + ", retry: " + str(count) + ", error:" + exstr)
 pass

 if data == None:
 log(" >> failed to download " + url)
 return

 #print data
 soup = BeautifulSoup(data)

 topTile = "[置顶]"
 titleDocs = soup.find_all("div", "article_title")
 for titleDoc in titleDocs:
 titleStr = titleDoc.a.get_text().encode('UTF-8')
 title = titleStr.replace(topTile, '').strip()
 #log(" >> title: " + title)

 manageDocs = soup.find_all("div", "article_manage")
 for managerDoc in manageDocs:
 categoryDoc = managerDoc.find_all("span", "link_categories")
 if len(categoryDoc) > 0:
 categories = categoryDoc[0].a.get_text().encode('UTF-8').strip()
 
 postDateDoc = managerDoc.find_all("span", "link_postdate")
 if len(postDateDoc) > 0:
 postDateStr = postDateDoc[0].string.encode('UTF-8').strip()
 postDate = datetime.datetime.strptime(postDateStr, '%Y-%m-%d %H:%M')

 contentDocs = soup.find_all(id="article_content")
 for contentDoc in contentDocs:
 htmlContent = contentDoc.prettify().encode('UTF-8')
 content = htmlContent2String(htmlContent)

 exportToMarkdown(output, postDate, categories, title, content)

def getPageUrlList(url):
 # 获取所有的页面的 url
 request = urllib2.Request(url, None, header)
 response = urllib2.urlopen(request)
 data = response.read()
 
 #print data
 soup = BeautifulSoup(data)

 lastArticleHref = None
 pageListDocs = soup.find_all(id="papelist")
 for pageList in pageListDocs:
 hrefDocs = pageList.find_all("a")
 if len(hrefDocs) > 0:
 lastArticleHrefDoc = hrefDocs[len(hrefDocs) - 1]
 lastArticleHref = lastArticleHrefDoc["href"].encode('UTF-8')

 if lastArticleHref == None:
 return []
 
 print " > last page href:" + lastArticleHref
 lastPageIndex = lastArticleHref.rfind("/")
 lastPageNum = int(lastArticleHref[lastPageIndex+1:])
 urlInfo = "http://blog.csdn.net" + lastArticleHref[0:lastPageIndex]

 pageUrlList = []
 for x in xrange(1, lastPageNum + 1):
 pageUrl = urlInfo + "/" + str(x)
 pageUrlList.append(pageUrl)
 log(" > page " + str(x) + ": " + pageUrl)

 log("total pages: " + str(len(pageUrlList)) + "\n")
 return pageUrlList

def getArticleList(url):
 # 获取所有的文章的 url/title
 pageUrlList = getPageUrlList(url)
 
 articleListDocs = []

 strPage = " > parsing page {0}"
 pageNum = 0
 global gRetryCount
 for pageUrl in pageUrlList:
 retryCount = 0
 pageNum = pageNum + 1
 pageNumStr = strPage.format(pageNum)
 print pageNumStr

 while retryCount <= gRetryCount:
 try:
 retryCount = retryCount + 1
 time.sleep(1.0) #访问太快会不响应
 request = urllib2.Request(pageUrl, None, header)
 response = urllib2.urlopen(request)
 data = response.read().decode('UTF-8')
 
 #print data
 soup = BeautifulSoup(data)
 
 topArticleDocs = soup.find_all(id="article_toplist")
 articleDocs = soup.find_all(id="article_list")
 articleListDocs = articleListDocs + topArticleDocs + articleDocs
 break
 except Exception, e:
 print "getArticleList exception:%s, url:%s, retry count:%d" % (e, pageUrl, retryCount)
 pass
 
 artices = []
 topTile = "[置顶]"
 for articleListDoc in articleListDocs:
 linkDocs = articleListDoc.find_all("span", "link_title")
 for linkDoc in linkDocs:
 #print linkDoc.prettify().encode('UTF-8')
 link = linkDoc.a
 url = link["href"].encode('UTF-8')
 title = link.get_text().encode('UTF-8')
 title = title.replace(topTile, '').strip()
 oneHref = "http://blog.csdn.net" + url
 #log(" > title:" + title + ", url:" + oneHref)
 artices.append([oneHref, title])

 log("total articles: " + str(len(artices)) + "\n")
 return artices

def getHtmlName(url):
 htmlNameIndex = url.rfind("/");
 urlLen = len(url)
 htmlName = ""
 if htmlNameIndex + 1 == urlLen:
 htmlNameIndex = url.rfind("/", 0, htmlNameIndex)
 htmlName = url[htmlNameIndex + 1:urlLen - 1]
 else:
 htmlName = url[htmlNameIndex + 1:]
 return htmlName

def exportBlog(username, output):
 url = "http://blog.csdn.net/" + username
 outputDir = output + "/" + username

 log(" >> user name: " + username)
 log(" >> output dir: " + outputDir)
 log("start export...")

 outputDir.replace("\\", "/")
 if not os.path.exists(outputDir.decode("utf-8")):
 os.makedirs(outputDir.decode("utf-8"))

 articleList = getArticleList(url)
 totalNum = len(articleList)

 log("start downloading...")
 currentNum = 0
 strPage = "[{0}/{1}] ".decode("utf-8").encode("utf-8")
 for article in articleList:
 currentNum = currentNum + 1
 strPageTemp = strPage.format(currentNum, totalNum)
 strPageTemp = strPageTemp + article[1]
 #log(strPageTemp)

 download(article[0], username)

log("============================================================")
exportBlog(__username__, __output__)
#download(__testArticleUrl__, __output__)