我正在尝试解析从 zillow 中选择两个下拉列表时填充的不同属性链接。完成选项的选择后，我可以在开发工具的 json 中看到结果。然而，当我使用下面的脚本做同样的事情时，我得到了一些奇怪的文本。
手动操作：
这就是我尝试自动化的方式：
import json
import requests
from pprint import pprint
link = 'https://www.zillow.com/search/GetSearchPageState.htm?'
params = {
'searchQueryState': {"pagination":{},"usersSearchTerm":"Vista, CA","mapBounds":{"west":-117.44051346728516,"east":-116.99488053271484,"south":33.126944633035116,"north":33.27919773006566},"regionSelection":[{"regionId":41517,"regionType":6}],"isMapVisible":True,"filterState":{"doz":{"value":"6m"},"isForSaleByAgent":{"value":False},"isForSaleByOwner":{"value":False},"isNewConstruction":{"value":False},"isForSaleForeclosure":{"value":False},"isComingSoon":{"value":False},"isAuction":{"value":False},"isPreMarketForeclosure":{"value":False},"isPreMarketPreForeclosure":{"value":False},"isRecentlySold":{"value":True},"isAllHomes":{"value":True},"hasPool":{"value":True},"hasAirConditioning":{"value":True},"isApartmentOrCondo":{"value":False}},"isListVisible":True,"mapZoom":11},
'wants': {"cat1":["listResults","mapResults"]},
'requestId': 2
}
with requests.Session() as s:
s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
res = s.get(link,params=json.dumps(params))
pprint(res.content)
这是它产生的输出：
b'<!-- This page outputs JSON instead of anything written here. -->'
如何使用请求从 zillow 解析自定义属性链接？
您必须对查询字符串进行编码，因为它出现在request URL.
为此，您需要：
urllib.parse.urlencode()
这是一个工作示例：
import json
import urllib.parse
import requests
link = 'https://www.zillow.com/search/GetSearchPageState.htm?'
params = {
'searchQueryState': {
"pagination": {},
"usersSearchTerm": "Vista, CA",
"mapBounds": {
"west": -117.44051346728516,
"east": -116.99488053271484,
"south": 33.126944633035116,
"north": 33.27919773006566
},
"regionSelection": [{"regionId": 41517, "regionType": 6}],
"isMapVisible": True,
"filterState": {
"doz": {"value": "6m"}, "isForSaleByAgent": {"value": False},
"isForSaleByOwner": {"value": False}, "isNewConstruction": {"value": False},
"isForSaleForeclosure": {"value": False}, "isComingSoon": {"value": False},
"isAuction": {"value": False}, "isPreMarketForeclosure": {"value": False},
"isPreMarketPreForeclosure": {"value": False},
"isRecentlySold": {"value": True}, "isAllHomes": {"value": True},
"hasPool": {"value": True}, "hasAirConditioning": {"value": True},
"isApartmentOrCondo": {"value": False}
},
"isListVisible": True,
"mapZoom": 11
},
'wants': {"cat1": ["listResults"]},
'requestId': 2
}
with requests.Session() as s:
s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
s.headers["x-requested-session"] = "BE6D8DA620E60010D84B55EB18DC9DC8"
s.headers["cookie"] = f"JSESSIONID={s.headers['x-requested-session']}"
data = json.dumps(
json.loads(s.get(f"{link}{urllib.parse.urlencode(params)}").content),
indent=2
)
print(data)
输出：
{
"user": {
"isLoggedIn": false,
"hasHousingConnectorPermission": false,
"savedSearchCount": 0,
"savedHomesCount": 0,
"personalizedSearchGaDataTag": null,
"personalizedSearchTraceID": "607a9ecb5aabe489c361c1d91f368b37",
"searchPageRenderedCount": 0,
"guid": "33b7add3-bfd3-4d85-a88a-d9d99256d2a2",
"zuid": "",
"isBot": false,
"userSpecializedSEORegion": false
},
"mapState": {
"customRegionPolygonWkt": null,
"schoolPolygonWkt": null,
"isCurrentLocationSearch": false,
"userPosition": {
"lat": null,
"lon": null
},
"regionBounds": {
"north": 33.275284,
"east": -117.145153,
"south": 33.130865,
"west": -117.290241
}
},
and much much more ...
注意：在该站点上放轻松，因为他们有非常敏感的反机器人措施，如果您继续过快地请求数据，他们会向您发送 CAPTCHA。
本文收集自互联网，转载请注明来源。
如有侵权，请联系 [email protected] 删除。
我来说两句