import requests                         
from bs4 import BeautifulSoup            
from urllib.parse import urlparse        
import time
import random

# made by jay1an 2023/9/12
def baidu_search(pages):
    Subdomains = []
    Subdomains_dict = {}    # 字典存储网域和标题
    # site:[域名]           eg. site:qq.com
    hearders = {                                                                 # 在这种情况下 仅加User-Agent即可正常获得数据
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.56',
    }
    for start in range(pages):
        url=f'https://www.baidu.com/s?wd=site%3Adouban.com&pn={start*10}'  # 改变参数实现翻页        # 该url为示例，爬取的是douban.com域名下的子域名
        resp = requests.get(url,headers=hearders)
        time.sleep(random.uniform(0.5,1.5))                                    # 加入随机延迟 模拟真实点击 (important)
        soup = BeautifulSoup(resp.content,'html.parser')                       # 使用BeautifulSoup对网页结果进行解析
        class_name = "result c-container xpath-log new-pmd"
        div_elements = soup.find_all('div', class_=class_name)                 # 找出class的值为result c-container xpath-log new-pmd的div块
        for i in div_elements:                                                 # 从div块中提取出url和title
            link = i.get('mu')                                                 
            domain = str(urlparse(link).scheme + "://" + urlparse(link).netloc)
            if domain in Subdomains:                                           # 去重
                pass
            else:
                Subdomains.append(domain)
                Subdomains_dict[domain] = i.h3.a.get_text()
    return Subdomains_dict                                                     # 返回值为字典，'domain' : 'title' 


# bing_search()

dict = baidu_search(10)           # 爬取10页的结果
for key, value in dict.items():   # 输出爬取的结果
    print(f'{key}: {value}')      # eg. "https://www.qq.com: 腾讯网"