爬虫：xpath模块及昵图网实例

xpath模块

from lxml import etree

str1 = """
<div>
    <ul>
         <li class="item-0"><a href="link1.html">first item</a></li>
         <li class="item-1"><a href="link2.html">second item</a></li>
         <li class="item-inactive"><a href="link3.html"><span class="bold">third item</span></a></li>
         <li class="item-1"><a href="link4.html">fourth item</a></li>
         <li class="item-0"><a class="test" href="link5.html">fifth item</a>111</li>
         <li class="item-0" vmid="10023898942">bsajkfhsdbfhjsdgbhfds</li>
     </ul>
 </div>
"""
# 将字符串转化为html对象
strtree1 = etree.html(str1)
print(strtree1.xpath('//li/text()')) #获取所有li里面的内容，这里只能获取直属li里面的内容
print(strtree1.xpath('//li/@class'))
# xpath中的下标，是从1开始的
print(strtree1.xpath('//li[3]/a/@href'))
# xpath中没有负数下标，若从右向左取，利用last获取最后一个 通过减一减二的方式获取倒数第二个倒数第三个
print(strtree1.xpath('//li[last()-1]/text()'))
# 获取指定属性值的标签
print(strtree1.xpath('//li[@class="item-1"]//text()'))

昵图网实例

from requests_html import htmlsession
from lxml import etree

session = htmlsession()
url = 'https://soso.nipic.com/?q=%e7%be%8e%e5%a5%b3'

response = session.get(url)
html = etree.html(response.text)

imgnames = []
imgurls = []
for i in range(1, 6):
    img_name = html.xpath(f'//ul[@id="img-list-outer"]/li[{i}]/a/img/@alt')
    img_url = html.xpath(f'//ul[@id="img-list-outer"]/li[{i}]/a/img/@data-original')
    imgnames.append(img_name[0])
    imgurls.append('https:' + img_url[0])

for i in range(len(imgurls)):
    with open('美女\\' + imgnames[i] + '.jpg', 'wb') as f:
        f.write(session.get(imgurls[i]).content)  # content就是将响应转化为二进制内容