requests 基础操作

requests是一个基于网络请求的模块。可以使用程序模拟浏览器上网。

环境安装

pip install requests

编码流程

案例 1 最简单的 request 请求

东方财富首页数据爬取

import requests

# 1.指定要爬取的地址
main_url = "https://www.eastmoney.com/"

# 2.发起请求
response = requests.get(main_url)

# 3.获取响应数据
response.encoding = "utf-8"
print(response.text)

# 4.保存数据
with open('dongfang.html', 'w', encoding='utf-8') as f:
    f.write(response.text)

案例 2 传参

爬取51游戏中任何游戏对应的搜索结果页面数据

import requests

#1.指定url
game_title = input('enter a game name:')
params = { #字典是用于封装请求参数
    'q':game_title
}
url = 'https://game.51.com/search/action/game/'

#2.发起请求
    #get是基于指定的url和携带了固定的请求参数进行请求发送
response = requests.get(url=url,params=params)

#3.获取响应数据
page_text = response.text #text表示获取字符串形式的响应数据
# print(page_text)

#4.持久化存储
fileName = game_title + '.html'
with open(fileName,'w') as fp:
    fp.write(page_text)

案例 3 User-Agent

中国人事考试网

直接请求会提示:很抱歉,由于您访问的URL有可能对网站造成安全威胁,您的访问被阻断。

import requests

# 1.指定要爬取的地址
main_url = "http://www.cpta.com.cn/"
#User-Agent:请求载体(浏览器,爬虫程序)的身份表示
header = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
}

# 2.发起请求
response = requests.get(main_url, headers=header)

# 3.获取响应数据
response.encoding = "utf-8"
print(response.text)

# 4.保存数据
with open('cpta.html', 'w', encoding='utf-8') as f:
    f.write(response.text)

中国人事考试网---站内搜索

  import requests
  url = 'http://www.cpta.com.cn/category/search.html'
  param = {
      "keywords": "人力资源",
      "搜 索": "搜 索"
  }
  header = {
      'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
  }
  #发起了post请求:通过data参数携带了请求参数
  response = requests.post(url=url,data=param,headers=header)
  
  page_text = response.text
  
  with open('renshi.html','w') as fp:
      fp.write(page_text)
  
  #通过抓包工具定位了指定的数据包:
      #提取:url,请求方式,请求参数,请求头信息

案例 4

智慧职教(动态加载数据爬取)(重点)

import requests
import time
fp = open('data.txt','w')
for page in range(1,6):
  time.sleep(1)
  url = 'https://www.icve.com.cn/portal/course/getNewCourseInfo?page='+str(page)
  headers = {
          'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
          'Referer':'https://www.icve.com.cn/portal_new/course/course.html'
  }
  dic = {
          "order": "",
          "kczy": "",
          "keyvalue": "",
          "printstate": ""
  }
  response = requests.post(url=url,headers=headers,data=dic)
  dic = response.json()
  for item_dic in dic['list']:
          course_title = item_dic['Title']
          teacher_name = item_dic['TeacherDisplayname']
          school_title = item_dic['UnitName']
          print(course_title,teacher_name,school_title)
          fp.write(course_title+'-'+teacher_name+'-'+school_title+'\n')

fp.close()

案例 5

图片数据爬取

#方式1:
import requests
url = 'https://img0.baidu.com/it/u=540025525,3089532369&fm=253&fmt=auto&app=138&f=JPEG?w=889&h=500'
response = requests.get(url=url)
#content获取二进制形式的响应数据
img_data = response.content
with open('1.jpg','wb') as fp:
  fp.write(img_data)
#方式2
from urllib.request import urlretrieve
#图片地址
img_url = 'https://img0.baidu.com/it/u=4271728134,3217174685&fm=253&fmt=auto&app=138&f=JPEG?w=400&h=500'
#参数1:图片地址
#参数2:图片存储路径
#urlretrieve可以根据图片地址将图片数据请求到直接存储到参数2表示的图片存储路径中
urlretrieve(img_url,'1.jpg')

案例6

肯德基(POST请求、动态加载数据、UA检测)

import requests
head = { #存放需要伪装的头信息
  'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
#post请求的请求参数
data = {
  "cname": "",
  "pid": "",
  "keyword": "天津",
  "pageIndex": "1",
  "pageSize": "10",
}
#在抓包工具中:Form Data存放的是post请求的请求参数,而Query String中存放的是get请求的请求参数
url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
#在post请求中,处理请求参数的是data这个参数不是params
response = requests.post(url=url,headers=head,data=data)
#将响应数据进行反序列化
page_text = response.json()
for dic in page_text['Table1']:
  name = dic['storeName']
  addr = dic['addressDetail']
  print(name,addr)

案例 7

通过抓包工具的分析发现,搜索菜谱的数据包有两个请求参数:

import requests
    
#请求头
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
}
title = input('请输入菜名:')
params = {
    'keyword':title,
    'cat':'1001'
}
#1.指定url
url = 'https://www.xiachufang.com/search/'

#2.发起请求
response = requests.get(url=url,headers=headers,params=params)
#处理乱码
response.encoding = 'utf-8' #gbk

#3.获取响应数据
page_text = response.text

#4.持久化存储
fileName = title + '.html'
with open(fileName,'w') as fp:
    fp.write(page_text)

案例 8

简历数据爬取

https://sc.chinaz.com/jianli/free.html

import requests
from lxml import etree

url = 'https://sc.chinaz.com/jianli/free.html'
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
}

response = requests.get(url=url,headers=headers)
response.encoding = "utf-8"
page_text = response.text

#数据解析:简历名称+详情页的url
tree = etree.HTML(page_text)
div_list = tree.xpath('//*[@id="container"]/div')

for div in div_list:
    title = div.xpath('./p/a/text()')[0]+'.rar'
    print(title)
    detail_url = div.xpath('./p/a/@href')[0]
    print(detail_url)
    #对详情页的url发起请求
    detail_page_response = requests.get(url=detail_url,headers=headers)
    detail_page_response.encoding = "utf-8"
    detail_page_text = detail_page_response.text
    # print(detail_page_text)
    tree = etree.HTML(detail_page_text)
    download_url = tree.xpath('//*[@id="down"]/div[2]/ul/li[1]/a/@href')[0]
    #在下载请求建立模板
    data = requests.get(url=download_url,headers=headers).content
    with open(title,'wb') as fp:
        fp.write(data)
    print(title,'保存下载成功!')
    # print("******")

图片懒加载:

案例 9

import requests
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
url = 'https://sc.chinaz.com/tupian/meinvtupian.html'
page_text = requests.get(url=url,headers=headers).text

tree = etree.HTML(page_text)
div_list = tree.xpath('/html/body/div[3]/div[2]/div')
for div in div_list:
    src = 'https:'+div.xpath('./img/@data-original')[0]
    print(src)

防盗链

  1. 在解析图片地址的时候,定位src的属性值,返回的内容和开发工具Element中看到的不一样,通过network查看网页源码发现需要解析real_src的值。
  2. 直接请求real_src请求到的图片不显示,加上Refere请求头即可

哪里找Refere:抓包工具定位到某一张图片数据包,在其requests headers中获取

import requests
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
    "Referer": "http://blog.sina.com.cn/",

}
url = 'http://blog.sina.com.cn/s/blog_01ebcb8a0102zi2o.html?tj=1'
page_text = requests.get(url,headers=headers).text
tree = etree.HTML(page_text)
img_src = tree.xpath('//*[@id="sina_keyword_ad_area2"]/div/a/img/@real_src')
for src in img_src:
    data = requests.get(src,headers=headers).content
    with open('./123.jpg','wb') as fp:
        fp.write(data)
    # break

视频数据爬取

import requests
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Referer':'https://www.51miz.com/'
}
url = 'https://www.51miz.com/shipin/'
response = requests.get(url=url,headers=headers)
page_text = response.text

#数据解析
tree = etree.HTML(page_text)
div_list = tree.xpath('/html/body/div[2]/div[2]/div[1]/div[2]/div[2]/div')
for div in div_list:
    src_list = div.xpath('./a/div/div/div/video/source/@src')
    #要给视频地址进行补全
    for src in src_list:
        src = 'https:' + src
        video_data = requests.get(url=src,headers=headers).content
        video_title = src.split('/')[-1]
        with open(video_title,'wb') as fp:
            fp.write(video_data)
        print(video_title,'爬取保存成功!')

    break

模拟登录

import requests
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
}
session = requests.Session()
data = {
    "userName": "bb328410948",
    "password": "bb328410948"
}
session.post('http://download.java1234.com/user/login',data=data,headers=headers)

page_text = session.get('http://download.java1234.com/toUserCenterPage',headers=headers).text

with open('1.html','w') as fp:
    fp.write(page_text)

代理(重要)

图片展示了代理服务器转发请求和响应的流程。左侧为浏览器,中间是代理服务器,右侧是Web server。浏览器向代理发送request,代理将request转发给Web server,Web server返回response给代理,代理再将response返回给浏览器。同时,浏览器向代理发送request,代理将request转发给Web server,Web server返回response给代理,代理将response返回给浏览器。该图直观呈现了代理服务器在请求和响应过程中的转发作用,与上下文介绍的代理服务器转发请求和响应的内容相契合。

import requests
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36',
}
url = 'http://www.cip.cc/'

page_text = requests.get(url,headers=headers).text
tree = etree.HTML(page_text)
text = tree.xpath('/html/body/div/div/div[3]/pre/text()')[0]
print(text.split('\n')[0])
import requests
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36',
}
url = 'http://www.cip.cc/'

page_text = requests.get(url,headers=headers,proxies={'http':'121.234.12.62:4246'}).text
tree = etree.HTML(page_text)
text = tree.xpath('/html/body/div/div/div[3]/pre/text()')[0]
print(text.split('\n')[0])
import requests
from lxml import etree
import random
import time

headers = {
  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36',
}
#代理池函数
def get_proxies_pool(url):
  ret_dic = requests.get(url=url).json()
  # 构建代理池
  proxies_list = []  # 存储多个代理服务器
  for dic in ret_dic['data']:
      ip = dic['ip']
      port = dic['port']
      dic = {}
      dic['https'] = str(ip) + ':' + str(port)
      proxies_list.append(dic)
  return proxies_list

#用来构建代理池
url = 'http://webapi.http.zhimacangku.com/getip?neek=b090cde57620cd17&num=5&type=2&time=4&pro=0&city=0&yys=0&port=11&pack=0&ts=0&ys=0&cs=0&lb=1&sb=&pb=4&mr=3&regions=&cf=0'
proxy_pool = get_proxies_pool(url)

for page in range(1, 5001):
  print('正在爬取第%d页的数据......' % page)
  #生成不同页码对应的url
  url = 'https://wz.sun0769.com/political/index/politicsNewest?id=1&page=%d' % page
  
  # 从代理池中随机选出一个代理
  proxy = random.choice(proxy_pool)
  
  page_text = requests.get(url=url, headers=headers,proxies=proxy).text
  time.sleep(0.5)
  tree = etree.HTML(page_text)
  #随便解析标题内容
  li_list = tree.xpath('/html/body/div[2]/div[3]/ul[2]/li')
  for li in li_list:
      title = li.xpath('./span[3]/a/text()')[0]
      print(title)