基于百度AI的自然语言处理文字分类

前言：

需要在百度AI平台注册登录并创建项目。

爬虫代码

import scrapy
from BaiDuAi.items import  BaiduaiItem

class AiSpider(scrapy.Spider):
    name = 'ai'
    # allowed_domains = ['www.xxx.com']
    #人民网url
    start_urls = ['http://politics.people.com.cn/n1/2018/1217/c1001-30470023.html']

    def parse(self, response):
        title=response.xpath('/html/body/div[4]/h1/text()').extract_first()
        content=response.xpath('//*[@id="rwb_zw"]//text()').extract()
        content=''.join(content).strip('\n \t')
        item=BaiduaiItem()
        item['title']=title
        item['content']=content

        yield  item

管道代码

from aip import AipNlp

""" 你的 APPID AK SK """
APP_ID = '15198150'
API_KEY = 'jaObSr6rmSmqsjWfKGGpmwxB'
SECRET_KEY = '808Eiz4FPkfMwS2ajClXYhKrcFMN1YUN'

client = AipNlp(APP_ID, API_KEY, SECRET_KEY)

class BaiduaiPipeline(object):
    keys=[]
    def process_item(self, item, spider):
        title=item['title'].replace('\xa0','')
        content=item['content'].replace('\xa0','')
        keys_dict=client.keyword(title,content)
        for dic in keys_dict['items']:
            self.keys.append(dic['tag'])


        keys="/".join(self.keys)
        typec_dic=client.topic(title,content)
        news_type=typec_dic['item']['lv1_tag_list'][0]['tag']

        with open('./xinwen.html','w',encoding='utf-8')as fp:
            fp.write(title+'\n\n'+content+'\n\n'+keys+'\n\n'+news_type)
        return item