python爬虫：抓取新浪新闻内容（从当前时间到之前某个时间段），并用jieba分词，用于训练自己的分词模型...

新浪新闻内容采用的是ajax动态显示内容，通过抓包，发现如下规律：

每次请求下一页，js那一栏都会出现新的url：

"http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
     "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
     "format=json&page=1&callback=newsloadercallback"

"http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
     "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
     "format=json&page=3&callback=newsloadercallback"

以上两个不同的是：page=？

Python实现代码：

#-*- coding:utf-8 -*-
__author__ = 'Administrator'

import re
from bs4 import BeautifulSoup
import urllib.request
import jieba
import string
import urllib.parse
from urllib.error import  HTTPError,URLError
import json

def get_page(num):
   return ("http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
     "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
     "format=json&page={}&callback=newsloadercallback").format(str(num))

def get_url(page_url):
    #解决请求路径中含义中文或特殊字符
    page_url=urllib.parse.quote(page_url, safe=string.printable)
    #print(page_url)
    url_list=[]
    try:
        res = urllib.request.urlopen(page_url)
    except HTTPError as e:
        print('The server couldn\'t fulfill the request.')
        print('Error code: ', e.code)
        return url_list
    except URLError as e:
        print('We failed to reach a server.')
        print('Reason: ', e.reason)
        return url_list
    else:
        if res.getcode()==200:
            jsdata=res.read().decode("utf-8")
            '''
            截取url方法一
            '''
            # result=re.findall('"url":"http.*?\.s?html"',jsdata)#.*后面再加？就可以变成非贪婪模式
            # for url in result:
            #     url=url.split(":",maxsplit=1)[1]
            #     url=url.replace('\\',"")
            #     url_list.append(url)
            '''
            截取url方法二
            '''
            data=jsdata[21:-2]
            data=re.sub('\'','\"',data)
            data=re.sub(r"\\u","",data)
            jsondata=json.loads(data)
            for dat in jsondata["result"]["data"]:
                url_list.append(dat["url"])
            return url_list



def get_context(new_url):
    #解决请求路径中含义中文或特殊字符
    httpurl=urllib.parse.quote(new_url, safe=string.printable)
    #print(httpurl)
    #print(type(httpurl))
    try:
        html=urllib.request.urlopen(httpurl)
    except HTTPError as e:
        print('The server couldn\'t fulfill the request.')
        print('Error code: ', e.code)
    except URLError as e:
        print('We failed to reach a server.')
        print('Reason: ', e.reason)
    else:
        if html.getcode()==200:
            res=html.read().decode("utf-8")
            #print(res)
            soup=BeautifulSoup(res,'html.parser')
            #print(soup.prettify)
            result={}
            result["article"]=''.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
            context=result['article']
            pattern='，|。|“|”|？|！|：|《|》|、|；|·|——| |‘|’|,|\?|\.|\!|`|~|\@|\#|\$|%|\^|\&|\*|（|）|\(|\)|-|\_|\+|=|\[|\]|\{|\}|"|\'|\<|\>|\||'
            li=re.split(pattern,context)
            #print("li")
            with open(r".\traindata.txt",'a',encoding='utf-8') as file:
                for l in li:
                    if l!="":
                        sentence = " ".join(jieba.cut(l))
                        file.write(sentence + '\n')

if __name__=="__main__":
    for i in range(1,1001):
        print("第 %d 页" % i)
        page_url=get_page(i)
        url_list=get_url(page_url)
        #print(url_list) #['"http://news.sina.com.cn/c/nd/2017-06-11/doc-ifyfzhac1171724.shtml"', ...],双引号外层还有单引号
        if url_list:
            for url in url_list:
                #print(eval(url))
                #print(type(url))
                # get_context(eval(url))#针对方法一截取url
                get_context(url)#针对方法二截取url