airbnb

有朋友要爬airbnb国内各大城市房源数据写论文，正好有空闲时间，所以就写了下，详情页用的是JS加载所以在获取详情页的时候采用了selenium模块,最下面代码展示了selenium获取加载后网页结构的写法，有兴趣的往下拉。

在搜索栏输入成时候，出现以下url，所以将需要的城市写成列表添加进红框位置即可切换城市

将该页面往下拉，翻页标签，获取翻页标签内的数字，并使用max函数获取的最大值，这样就得到了最大页面的值

获取详情页的url

访问这些详情url发现是用JS加载进来的，所以这里我们需要用到selenium+phantomjs去模拟浏览的形式去获取加载后的HTML，这里有个小技巧如果使用selenium获取加载后的HTML速度较慢，所以将加载完毕后的HTML保存到本地，用本地去调试抓取详情页。

分析HTML后，发现需要的详情信息在div#details里，其中通过div.col-md-9还分为几个信息块，这些信息块不是每个url都会展示，所以需要通过.about-this-listing-section-title去判断前面的块标签，再选取相应的数据抽取策略（下图是HTML标签特征）

HTML标签特征

好了，简单的分析就到这详细具体请看下面代码

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
        AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
    "Connection": "keep-alive",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.8"
}
city_tuple = 'Shanghai', 'beijing', 'chengdu', 'chongqing', \
             'hangzhou', 'guangzhou', 'nanjing', 'suzhou', \
             'xian', 'qingdao'
for city_name in city_tuple:
    city_url = 'https://zh.airbnb.com/s/{}--China?cdn_cn=1&s_tag=Ic3cziVn&allow_override%5B%5D='.format(city_name)
    response = requests.get(city_url, headers=headers)
    # print(response.text)
    soup = BeautifulSoup(response.content, 'lxml')
    soup.encode('utf-8')
    # print(soup)
    info_area = soup.select('div.cardGrid_o1rkka div a')
    href_list = [_.get("href") for _ in info_area]
    # print(href_list)
    all_room_info_url_list = ['https://zh.airbnb.com/' + _ for _ in href_list]

    page_num_list = soup.select('div > ul.buttonList_11hau3k > li')
    max_page = max([int(_.get_text().strip()) for _ in page_num_list if _.get_text().strip().isdigit()])
    # print(max_page)
    for pg in range(1, max_page):
        city_url_pg = city_url + '& section_offset = {}'.format(pg)
        response = requests.get(city_url_pg, headers=headers)
        info_area = soup.select('div.cardGrid_o1rkka div a')
        href_list = [_.get("href") for _ in info_area]
        room_info_url_list = ['https://zh.airbnb.com/' + _ for _ in href_list]
        all_room_info_url_list.extend(room_info_url_list)
    # 城市及该城市下所有url
    print(city_name, all_room_info_url_list)

    # 详情页
    for info_url in all_room_info_url_list:
        print('详情页url', info_url)
        selenium_ua = headers["User-Agent"]
        options = dict(DesiredCapabilities.PHANTOMJS)
        options["phantomjs.page.settings.userAgent"] = selenium_ua
        browser = webdriver.PhantomJS(desired_capabilities=options)
        browser.get(info_url)
        browser.implicitly_wait(60)
        soup = BeautifulSoup(browser.page_source, 'lxml')
        browser.close()

        room_sorce_list = soup.select('div#details div.col-md-9')
        # print(len(room_sorce_list))

        # 评分
        star_rate = soup.select_one('div.star-rating')
        if star_rate is not None:
            star_rate = star_rate.get('content')
        print(star_rate)
        # 每日价格
        price = soup.select_one('span#book-it-price-string')
        if price is not None:
            price = price.get_text()
        print(price)

        section_title_list = [_.get_text() for _ in soup.select('.about-this-listing-section-title')]
        #       # 取出披露的项目列表
        # print(section_title_list)

        for idx_num, idx_title in enumerate(section_title_list):
            print(idx_num, idx_title)
            if idx_title == '商务差旅':
                for i in room_sorce_list[idx_num].select('span > span'):
                    # 商务差旅
                    print(i.get_text())
            if idx_title == '房源':
                for i in room_sorce_list[idx_num].select('div.bottom-spacing-2'):
                    # 房源信息
                    print(i.get_text())
            elif idx_title == '便利设施':
                print('**可使用设施**')
                for i in room_sorce_list[idx_num].select('strong'):
                    # 可使用设施
                    print(i.get_text())
                print('**不可使用设施**')
                for i in room_sorce_list[idx_num].select('del'):
                    # 不可使用设施
                    print(i.get_text())
            elif idx_title == '收费相关' or idx_title == '相关收费':
                for i in room_sorce_list[idx_num].select('div.bottom-spacing-2'):
                    # 收费相关
                    print(i.get_text())
            elif idx_title == '床位安排':
                for i in room_sorce_list[idx_num].select('div.text-muted'):
                    # 床位安排
                    print(i.get_text())
            elif idx_title == '《房屋守则》':
                for i in room_sorce_list[idx_num].select('div.col-sm-12'):
                    # 《房屋守则》
                    print(i.get_text())
            elif idx_title == '预订取消政策':
                for i in room_sorce_list[idx_num].select('p'):
                    # 预订取消政策
                    print(i.get_text())
            elif idx_title == '安全设施':
                for i in room_sorce_list[idx_num].select('span'):
                    # 安全设施
                    print(i.get_text())
            elif idx_title == '可订状态':
                for i in room_sorce_list[idx_num].select('div.bottom-spacing-2')[:-1]:
                    # 可订状态
                    print(i.get_text())