Customization of request object
import urllib.request
url = 'https://www.baidu.com'
# Composition of url
# https://www.baidu.com/s?wd=Jay Chou
# http/https www.baidu.com 80/443 s wd = Jay Chou #
# Protocol host port number path parameter anchor point
#http80
#https443
# mysql 3306
#oracle 1521
#redis 6379
#mongodb 27017
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
# Because dictionaries cannot be stored in the urlopen method, headers cannot be passed in
# Request object customization
request = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(request)
content = response. read(). decode('utf8')
print(content)
quote method of get request
# https://www.baidu.com/s?wd=Jay Chou
# Requirement Get https://www.baidu.com/s?wd=Jay Chou's webpage source code
import urllib.request
import urllib.parse
url = 'https://www.baidu.com/s?wd='
# The customization of the request object is the first means to solve anti-crawling
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
# Convert the three characters of Jay Chou into unicode format
# We need to depend on urllib.parse
name = urllib.parse.quote('Jay Chou')
url = url + name
# Request object customization
request = urllib.request.Request(url=url,headers=headers)
# Simulate the browser to send a request to the server
response = urllib.request.urlopen(request)
# Get the content of the response
content = response.read().decode('utf-8')
#Print data
print(content)
urlencode method of get request
# urlencode application scenario: when there are multiple parameters
# https://www.baidu.com/s?wd=Jay Chou & amp;sex=male
# import urllib.parse
#
#data = {
# 'wd':'Jay Chou',
# 'sex':'male',
# 'location':'Taiwan Province, China'
# }
#
# a = urllib.parse.urlencode(data)
# print(a)
#Get the source code of https://www.baidu.com/s?wd=Jay Chou & amp;sex=male
import urllib.request
import urllib.parse
base_url = 'https://www.baidu.com/s?'
data = {
'wd':'Jay Chou',
'sex':'male',
'location':'Taiwan Province, China'
}
new_data = urllib.parse.urlencode(data)
# Request resource path
url = base_url + new_data
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
# Request object customization
request = urllib.request.Request(url=url,headers=headers)
# Simulate the browser to send a request to the server
response = urllib.request.urlopen(request)
# Get the data of the web page source code
content = response.read().decode('utf-8')
#Print data
print(content)
post request Baidu translation
# post request
import urllib.request
import urllib.parse
url = 'https://fanyi.baidu.com/sug'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}
data = {
'kw':'spider'
}
# The parameters of the post request must be encoded
data = urllib.parse.urlencode(data).encode('utf-8')
# The parameters of the post request will not be spliced at the end of the url, but need to be placed in the parameters customized by the request object.
# The parameters of the post request must be encoded
request = urllib.request.Request(url=url,data=data,headers=headers)
# Simulate a browser sending a request to the server
response = urllib.request.urlopen(request)
# Get response data
content = response. read(). decode('utf-8')
# String -- "json object
import json
obj = json. loads(content)
print(obj)
# The parameters of the post request method must be encoded data = urllib.parse.urlencode(data)
# After encoding, the encode method must be called data = urllib.parse.urlencode(data).encode('utf-8')
# The parameters are placed in the request object customization method request = urllib.request.Request(url=url,data=data,headers=headers)
post request for detailed translation from Baidu Translation
import urllib.request
import urllib.parse
url = 'https://fanyi.baidu.com/v2transapi?from=en & amp;to=zh'
headers = {
# 'Accept': '*/*',
# 'Accept-Encoding': 'gzip, deflate, br',
# 'Accept-Language': 'zh-CN,zh;q=0.9',
# 'Connection': 'keep-alive',
# 'Content-Length': '135',
# 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'BIDUPSID=DAA8F9F0BD801A2929D96D69CF7EBF50; PSTM=1597202227; BAIDUID=DAA8F9F0BD801A29B2813502000BF8E9:SL=0:NR=10:FG=1; __yjs_duid=1_c19765bd 85fa6fa12c2853fc392f8db1618999058029; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH = 1; BDUSS = R2BezVTJFCNHQXDUV-CTZ-MZZRSGXHBUYWSKUWK2SKXXS3E2M2M2LQAFRLJOBQUFBJCQAAAAAAAAAAAAAAAAAAAAAAAA3E ~ 9shlzgf5aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAsedsedctctationationationationationntsntsist11bezvtjFCNHQXDUV-MZZRSG xhbuywskrkuw2skxxs3e2M2LQAFRLLJORLJOFBQUFBJCQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3E ~ 9shlzgf5aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaxgbtnoxgaw; BDORZ = B490B5EBF6F3CD402E515D22BCDA1598; Baiduid_bfess = DAA8F9F0BD801A29B2813502000 BF8E9: SL = 0: NR = 10: FG = 1; BDRCVFR [Fewj1vr5u3D ]=I67x6TjHwwYf0; PSINO=2; 1b61gi6ka20q; BCLID=10943521300863382545; BDSFRCVID=boDOJexroG0YyvRHKn7hh7zlD_weG7bTDYLEOwXPsp3LGJLVJeC6EG0Pts1-dEu-EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0 Ptf8g0M5; H_BDCLCKID_SF=tR3aQ5rtKRTffjrnhPF3-44vXP6-hnjy3bRkX4Q4Wpv_Mnndjn6SQh4Wbttf5q3RymJ42-39LPO2hpRjyxv4y4Ldj4oxJpOJ-bCL0p5aHl51fbbvbURvD- ug3-7qqU5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP -jBRIE3-oJqC8hMIt43f; BCLID_BFESS=10943521300863382545; BDSFRCVID_BFESS=boDOJexroG0YyvRHKn7hh7zlD_weG7bTDYLEOwXPsp3LGJLVJeC6EG0Pts1-dEu-EHtdogKK0mOTHv8F_2ux Ojjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tR3aQ5rtKRTffjrnhPF3-44vXP6-hnjy3bRkX4Q4Wpv_Mnndjn6SQh4Wbttf5q3RymJ42-39LPO2hpRjyxv4y4Ldj4oxJpOJ-b CL0p5aHl51fbbvbURvD-ug3-7qqU5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIE3-oJqC8hMIt43f; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1629701482,162 9702031, 1629702343, 1629704515; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1629704515; __yjs_st=2_MDBkZDdkNzg4YzYyZGU2NTM5NzBjZmQ0OTZiMWRmZGUxM2QwYzkwZTc2NTZmMmIxNDJkYz k4NzU1ZDUzN2U3Yjc4ZTJmYjE1YTUzMTljYWFkMWUwYmVmZGEzNmZjN2FlY2M3NDAzOThhZTY5NzI0MjVkMmQ0NWU3MWE1YTJmNGE5NDBhYjVlOWY3MTFiMWNjYTVhNWYWI0YThlMDUjODBk 2NjMwMzY2MjFhZDNkMzVhNGMzMGZkMWY2NjU5YzkxMDk3NTEzODJiZWUyMjEyYTk5YzY4ODUyYzNjZTJjMGM5MzhhMWE5YjU3NTM3NWZiOWQxNmU3MDVkODExYzFjN183XzliY2RhYj gz; ab_sr=1.0.1_ZTc2ZDFkMTU5ZTM0ZTM4MWVlNDU2MGEzYTM4MzZiY2I2MDIxNzY1Nzc1OWZjZGNiZWRhYjU5ZjYwZmNjMTE2ZjIzNmQxMTdiMzIzYTgzZjVjMTY0ZjM1YjMwZTdj MjhiNDRmN2QzMjMwNWRhZmUxYTJjZjZhNTViMGM2ODFlYjE5YTlmMWRjZDAwZGFmMDY4ZTFlNGJiZjU5YzE1MGIxN2FiYTU3NDgzZmI4MDdhMDM5NTQ0MjQxNDBiNzdhMDdl',
# 'Host': 'fanyi.baidu.com',
# 'Origin': 'https://fanyi.baidu.com',
# 'Referer': 'https://fanyi.baidu.com/?aldtype=16047',
# 'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
# 'sec-ch-ua-mobile': '?0',
# 'Sec-Fetch-Dest': 'empty',
# 'Sec-Fetch-Mode': 'cors',
# 'Sec-Fetch-Site': 'same-origin',
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36',
# 'X-Requested-With': 'XMLHttpRequest',
}
data = {
'from': 'en',
'to': 'zh',
'query': 'love',
'transtype': 'realtime',
'simple_means_flag': '3',
'sign': '198772.518981',
'token': '5483bfa652979b41f9c90d91f3de875d',
'domain': 'common',
}
# The parameters of the post request must be encoded and the encode method must be called.
data = urllib.parse.urlencode(data).encode('utf-8')
# Request object customization
request = urllib.request.Request(url = url,data = data,headers = headers)
# Simulate the browser to send a request to the server
response = urllib.request.urlopen(request)
# Get response data
content = response.read().decode('utf-8')
import json
obj = json.loads(content)
print(obj)