python爬虫获取豆瓣网前250部电影的详细信息

网址 https://movie.douban.com/top250

一共250部电影，有分页，获取每一部的详细信息

不采用框架，使用 urilib读取网页，re进行正则表达式匹配，lxml进行xpath查找

from film import *
from urllib import request
import time,re
url=r'https://movie.douban.com/top250?start='
for i in range(10):
    url=url+str(i*25)
    print(url)
    
    headers = {
        'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
        'Connection': 'keep-alive'
    }
    req=request.Request(url,headers=headers)
    page=request.urlopen(req).read()
    page=page.decode('utf-8')
    #fp=open("page.txt",mode="w",encoding="UTF-8")
    #fp.writelines(page)
    p=re.compile(r'\<em\sclass=\"\"\>\d+\</em\>\s*\<a\shref=\"https://movie.douban.com/subject/\d+/\"\>')
    result=p.findall(page)
    for item in result:
        #print(item)
        p=re.compile(r'\d+')
        no=p.findall(item)
        #print(no[0])
        p=re.compile(r'https://movie.douban.com/subject/\d+/')
        rurl=p.findall(item)
        #print(rurl[0])
        filma=film(no[0],rurl[0],'','','','','','')
        filma.getall()
        filma.detail()
        time.sleep(3)
    #print (result)
    time.sleep(3)
    #print(i)

 

film.py 如果要做数据的持久化，在这里实现

from urllib import request
from lxml import etree
class film:
    def __init__(self,no,url,name,year,score,director,classification,actor):
        self.name=name
        self.year=year
        self.score=score
        self.director=director
        self.classification=classification
        self.actor=actor
        self.url=url
        self.no=no
    
    def detail(self):
        temp = "No：%s;url：%s;片名：%s;年份：%s;分数：%s;导演：%s;分级：%s;演员：%s;"   %(self.no,self.url,self.name,self.year,self.score,self.director,self.classification,self.actor)  
        print(temp)
    def getall(self):
        headers={
        'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
        'Connection': 'keep-alive'
        }
        req=request.Request(self.url,headers=headers)
        page=request.urlopen(req).read()
        page=page.decode('utf-8')
        selector=etree.HTML(page)
        print (page)
        self.name=selector.xpath('/html/body/div[3]/div[1]/h1/span[1]/text()')
        self.year=selector.xpath('//*[@id="content"]/h1/span[2]/text()')
        self.score=selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')
        self.director=selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')
        self.classification=selector.xpath('//*[@id="info"]/span[5]/text()')
        self.actor=selector.xpath('//*[@id="info"]/span[3]/span[2]/a/text()')