需求背景：我们有自己的平台，但是希望在我们的平台上面想要实现一个百度搜索的接口，输入想要搜索的内容，模拟百度搜索，将返回的内容再展现在我们自己的平台中，提供给用户查看。
# -- coding:utf8 --import hashlibimport randomimport sysimport timefrom pprint import pprintfrom urllib.parse import quotefrom bs4 import BeautifulSoupfrom requests_html import HTMLSession# 代理ipclass GetProxy: _instance = None @staticmethod def get_random_proxy(): while not GetProxy._instance: GetProxy._instance = ['211.149.199.235:16818', '123.57.57.125:16818'] proxy = random.choice(GetProxy._instance).strip() return proxy """返回HTTP/HTTPS的代理IP，可根据代理IP类型更改""" def get_ip_http(self): proxies = { "http": self.get_random_proxy(), } return proxies def get_ip_https(self): proxies = { "https": self.get_random_proxy(), } return proxies# 随机获取一个cookiedef get_random_cookie(): cookies = [ 'PSTM=1645440125; BAIDUID=49D5966BB6F2D98A8378EC10151CE748:FG=1; BAIDUID_BFESS=49D5966BB6F2D98A8378EC10151CE748:FG=1; BIDUPSID=5C48EADF0E27C74CB11F290539E5EAA8; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; __yjs_duid=1_6b058121c11c500f39afbc042ec623711645440178604; delPer=0; PSINO=7; MCITY=-257%3A; BA_HECTOR=05a0ak0ga42525a5us1h18lb30r; BDRCVFR[C0p6oIjvx-c]=rJZwba6_rOCfAF9pywd; H_PS_PSSID=35105_35865_34584_35491_35872_35246_35319; ab_sr=1.0.1_ZGM2MTQ3YjE2NGE0ZmE2NWNhNGYzMDQ1Nzg1ZWYxYWFjZDllZjA1NzY0YWE3NjVjZmEyNjA4NmE5NTljZTEzOTFkNzViMWRlNTA4ZmQwYWIzYWZlYjQyMDYxZTcxNGI0NWVjYzU5ODk0ZDVmYmNkZDI4YzkyNGEwNTUwZjc4MWU3Y2Q0ZTUzOGExNjQwZTgzMzM4ZjQ2ZjkzMjE0OGNjZA==; BAIDU_WISE_UID=wapp_1645499858512_985', 'BIDUPSID=0AB15879656FD166028DF65039BDFF15; PSTM=1641442191; BAIDUID=911EF71E90573B2693EC612910B1F7BE:FG=1; BCLID_BFESS=9239639223377566883; BDSFRCVID_BFESS=1T-OJeCmHxdstirHc7RXbo9jumKK0gOTHllnPXllHP8_1buVJeC6EG0Ptf8g0KubFTPRogKK0gOTH6KF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tJkD_I_hJKt3fP36q6_a2-F_2xQ0etJXf5Txbp7F5lOVO-ngKU613MkSjNOj5t482jTLahkM5h7xObR1hl3ih-An0a7dJ4jtQeQ-5KQN3KJmfbL9bT3v5tDz3b3N2-biWbRM2MbdJqvP_IoG2Mn8M4bb3qOpBtQmJeTxoUJ25DnJhhCGe6-MjT3-DG8jqbvEHDc-WJ3t-TrjDCvRhMjcy4LdjG5N0PJT5bv73K022boobJcGLqjW0R_X3-Aq54RMagQwLPJEytQTS-5VbtoMQfbQ0-cOqP-jWbnu-qTo2n7JOpkRbUnxy50vQRPH-Rv92DQMVU52QqcqEIQHQT3m5-5bbN3ht6IHJbCJoDD5tIvbfP0kjjQWMt_h-fuX5-CstGPL2hcH0b61JbbR5-rKy-JW0R7a25cBbCjiaKJjBMb1DbRk0h7ShMkrebPD5JQpWDTm_q5TtUJMeCnTDMRh-xK70b5yKMnitIv9-pPKWhQrh459XP68bTkA5bjZKxtq3mkjbPbDfn028DKu-n5jHj3WDG-J3q; __yjs_duid=1_ada3d0ac8d4be7042dd53d52221555631641452261829; BAIDUID_BFESS=911EF71E90573B2693EC612910B1F7BE:FG=1; BD_HOME=1; H_PS_PSSID=35104_31660_34584_35490_35841_35887_35542_35318_26350_35867_22158; BD_UPN=12314753; delPer=0; BD_CK_SAM=1; PSINO=7; H_PS_645EC=09c89Z6QKcJ4xzJZr1LUqxrp0qdbpltyn/ixDDrfq5R6r0cQWwLiJT3HLZY; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BA_HECTOR=a424810gag04818hg31h15uop0q; baikeVisitId=492b5e23-3a27-4d6d-bf0a-ab5907361a87; BDSVRTM=643'] cooke = random.choice(cookies).strip() return cookedef search_kw(kw, pn, tp): cookie = get_random_cookie() headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", "Referer": "https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=2&ch=&tn=baiduhome_pg&bar=&wd=123&oq=123&rsv_pq=896f886f000184f4&rsv_t=fdd2CqgBgjaepxfhicpCfrqeWVSXu9DOQY5WyyWqQYmsKOC%2Fl286S248elzxl%2BJhOKe2&rqlang=cn", # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,**;q=0.8,application/signed-exchange;v=b3;q=0.9", "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7", "Sec-Fetch-Mode": "navigate", "cookie": cookie, "Connection": "Keep-Alive", } if cookie: if "__yjs_duid" not in cookie: pass else: _ = cookie.split("__yjs_duid=") __ = _[1].split(";", 1)[-1] ___ = hashlib.md5() cookie = _[0] + "__yjs_duid=1_" + str(___.hexdigest()) + __ headers["cookie"] = cookie + ";random=" + str(random.randint(500, 4000)) text = quote(kw, "utf-8") rsv_page = '' cp = 1 if pn == 'np': rsv_page = '&rsv_page=1' else: cp = int(pn) if tp == '1': # 网页 url = f"https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" elif tp == '2': # 图片 url = f"https://image.baidu.com/search/index?tn=baiduimage&ps=1&ct=201326592&lm=-1&cl=2&nc=1&ie=utf-8&word={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" # elif tp == '3': # 资讯 # url = f"https://www.baidu.com/s??rtt=1&bsst=1&cl=2&tn=news&word={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" # elif tp == '4': # 知道 # url = f"https://zhidao.baidu.com/search?ie=utf-8&f=8&rsv_bp=1&tn=baidu&word={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" # elif tp == '5': # 贴吧 # url = f"https://tieba.baidu.com/f?fr=wwwt&ie=utf-8&kw={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" # elif tp == '6': # 视频 # url = f"https://www.baidu.com/sf/vsearch?pd=video&tn=vsearch&ie=utf-8&wrsv_spt=10&wd={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" elif tp == '3': # 文库 url = f"https://wenku.baidu.com/search?lm=0&od=0&ie=utf-8&word={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" elif tp == '4': # 百科 url = f"https://baike.baidu.com/search?lm=0&od=0&ie=utf-8&word={kw}&oq={text}&pn={(cp - 1) * 10}&inputT={random.randint(500, 4000)}{rsv_page}" proxies = GetProxy().get_ip_http() session = HTMLSession() r = session.get(url, headers=headers, proxies=proxies) r.encoding = r.apparent_encoding content_html = r.html.html soup = BeautifulSoup(content_html, features="html.parser") # 删除无用元素 if tp == '1': # 网页 head = soup.find("div", id="head") if head: head.decompose() s_tab = soup.find("div", id="s_tab") if s_tab: s_tab.decompose() content_right = soup.find("div", id="content_right") if content_right: content_right.decompose() rs_new = soup.find("div", id="rs_new") if rs_new: rs_new.decompose() foot = soup.find("div", id="foot") if foot: foot.decompose() elif tp == '2': # 图片 head = soup.find("div", id="head") if head: head.decompose() s_tab = soup.find("div", id="s_tab") if s_tab: s_tab.decompose() content_right = soup.find("div", id="content_right") if content_right: content_right.decompose() rs_new = soup.find("div", id="rs_new") if rs_new: rs_new.decompose() foot = soup.find("div", id="foot") if foot: foot.decompose() bdpcImgTab = soup.find("div", id="bdpcImgTab") if bdpcImgTab: bdpcImgTab.decompose() topInfoBar = soup.find("div", id="topInfoBar") if topInfoBar: topInfoBar.decompose() # elif tp == '3': # 资讯 # head = soup.find("div", id="head") # if head: # head.decompose() # s_tab = soup.find("div", id="s_tab") # if s_tab: # s_tab.decompose() # content_right = soup.find("div", id="content_right") # if content_right: # content_right.decompose() # rs_new = soup.find("div", id="rs_new") # if rs_new: # rs_new.decompose() # foot = soup.find("div", id="foot") # if foot: # foot.decompose() # # # elif tp == '4': # 知道 # head = soup.find("div", id="head") # if head: # head.decompose() # s_tab = soup.find("div", id="s_tab") # if s_tab: # s_tab.decompose() # content_right = soup.find("div", id="content_right") # if content_right: # content_right.decompose() # rs_new = soup.find("div", id="rs_new") # if rs_new: # rs_new.decompose() # foot = soup.find("div", id="foot") # if foot: # foot.decompose() # right_billboard = soup.find("div", id="right-billboard") # if right_billboard: # right_billboard.decompose() # # # elif tp == '5': # 贴吧 # head = soup.find("div", id="head") # if head: # head.decompose() # s_tab = soup.find("div", id="s_tab") # if s_tab: # s_tab.decompose() # content_right = soup.find("div", id="content_right") # if content_right: # content_right.decompose() # rs_new = soup.find("div", id="rs_new") # if rs_new: # rs_new.decompose() # foot = soup.find("div", id="foot") # if foot: # foot.decompose() # pagelet_head = soup.find("div", id="pagelet_frs-header/pagelet/head") # if pagelet_head: # pagelet_head.decompose() # normal_aside = soup.find("div", id="pagelet_frs-aside/pagelet/normal_aside") # if normal_aside: # normal_aside.decompose() # content_footer = soup.find("div", id="pagelet_frs-footer/pagelet/content_footer") # if content_footer: # content_footer.decompose() # # # elif tp == '6': # 视频 # head = soup.find("div", id="head") # if head: # head.decompose() # s_tab = soup.find("div", id="s_tab") # if s_tab: # s_tab.decompose() # content_right = soup.find("div", id="content_right") # if content_right: # content_right.decompose() # rs_new = soup.find("div", id="rs_new") # if rs_new: # rs_new.decompose() # foot = soup.find("div", id="foot") # if foot: # foot.decompose() # con_ar = soup.find("div", id="con-ar") # if con_ar: # con_ar.decompose() elif tp == '3': # 文库 head = soup.find("div", id="head") if head: head.decompose() s_tab = soup.find("div", id="s_tab") if s_tab: s_tab.decompose() content_right = soup.find("div", id="content_right") if content_right: content_right.decompose() rs_new = soup.find("div", id="rs_new") if rs_new: rs_new.decompose() foot = soup.find("div", id="foot") if foot: foot.decompose() elif tp == '4': # 百科 head = soup.find("div", iclass_="wiki-common-headTab") if head: head.decompose() foot = soup.find("div", class_="wgt-footer-main") if foot: foot.decompose() page_content = soup.find("div", id="page") if page_content: links = page_content.findAll("a") if links: for link in links: span = link.find('span') cp = 'np' if span: cp = span.get_text() href = f"/article/searchEverything?wd={kw}&pn={cp}&tp={tp}" link['href'] = href soup_content = soup.prettify(encoding="utf-8", formatter='minimal') print(soup_content)if __name__ == '__main__': kw = '广州市区' pn = '1' tp = '4' if len(sys.argv) == 4: kw = sys.argv[1] pn = sys.argv[2] tp = sys.argv[3] elif len(sys.argv) == 3: pn = sys.argv[1] tp = sys.argv[2] result = search_kw(kw, pn, tp)