關(guān)于python爬蟲應(yīng)用urllib庫作用分析

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/2 19:24
# @FileName : testUrllib.py
# Software : PyCharm
import urllib.request
#get請求
response = urllib.request.urlopen("http://www.baidu.com")  #返回的是存儲網(wǎng)頁數(shù)據(jù)的對象
#print(response)  可以嘗試打印一下看一下
print(response.read().decode('utf-8')) #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/2 19:24
# @FileName : testUrllib.py
# Software : PyCharm
import urllib.request
response = urllib.request.urlopen("http://www.baidu.com")  #返回的是存儲網(wǎng)頁數(shù)據(jù)的對象
data = response.read().decode('utf-8')  #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼
#print(data)
with open("index.html",'w',encoding='utf-8') as wfile: #或者你們也可以常規(guī)打開，不過需要最后關(guān)閉記得close()
    wfile.write(data)
    print("讀取結(jié)束")

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/2 19:24
# @FileName : testUrllib.py
# Software : PyCharm
import urllib.request
import urllib.parse #解析器
data = bytes(urllib.parse.urlencode({"hello":"world"}),encoding='utf-8')  #轉(zhuǎn)換為二進制數(shù)據(jù)包，里面是鍵值對（有時輸入的用戶名：密碼就是這樣的），還有一些編碼解碼的數(shù)值等.這里就是按照utf-8的格式進行解析封裝生成二進制數(shù)據(jù)包
response = urllib.request.urlopen("http://httpbin.org/post",data=data)  #返回的請求
print(response.read().decode('utf-8'))      #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/2 19:24
# @FileName : testUrllib.py
# Software : PyCharm
import urllib.request
try:
    response = urllib.request.urlopen("http://httpbin.org/get",timeout=0.01)  #返回的是存儲網(wǎng)頁數(shù)據(jù)的對象, 直接用這個網(wǎng)址的get請求了.timeout表示超時，超過0.01秒不響應(yīng)就報錯，避免持續(xù)等待
    print(response.read().decode('utf-8'))      #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼
except urllib.error.URLError as e:
    print("超時了\t\t錯誤為：",e)

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/2 19:24
# @FileName : testUrllib.py
# Software : PyCharm
import urllib.request
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
}
request = urllib.request.Request("http://douban.com", headers=headers) #返回的是請求，將我們偽裝成瀏覽器發(fā)送的請求
response = urllib.request.urlopen(request)  #返回的是存儲網(wǎng)頁數(shù)據(jù)的對象
data = response.read().decode('utf-8')     #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼
with open("index.html",'w',encoding='utf-8') as wfile: #或者你們也可以常規(guī)打開，不過需要最后關(guān)閉記得close()
    wfile.write(data)

# -*- codeing = utf-8 -*-
# @Author: Y-peak
# @Time : 2021/9/3 0:47
# @FileName : testUrllib.py
# Software : PyCharm

import urllib.request
import urllib.parse
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
}
url = "http://httpbin.org/post"
data = (bytes)(urllib.parse.urlencode({"賬戶":"密碼"}),encoding = 'utf-8')
request = urllib.request.Request(url, data = data,headers=headers, method='POST') #返回的是請求
response = urllib.request.urlopen(request)  #返回的是存儲網(wǎng)頁數(shù)據(jù)的對象
data = response.read().decode('utf-8')     #通過read將數(shù)據(jù)讀取出來, 使用utf-8解碼防止有的地方出現(xiàn)亂碼
print(data)