采集器

查看采集器

属性	值	属性	值
名称	travel_booking	创建时间	2025-04-02 10:10:02
标题	旅游-booking	更新时间	-
状态	running	开始时间	2025-06-29 21:08:36
停止类型		结束时间	-
信号		运行时长	189.6小时
进程ID	121401	进程名称	Process-4
作业ID	832bade4-5572-4b87-85e6-eab9245f9fac	作业目录	records/jobs/832bade4-5572-4b87-85e6-eab9245f9fac
日志文件	records/logs/832bade4-5572-4b87-85e6-eab9245f9fac.log (查看日志) (tail日志)
采集器类型	scrapy	文件	travel_booking_spider.py
起始urls(每行一个) (若为空则使用代码中的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后条历史作业

代码
import os import platform import random import re import time import traceback from enum import Enum from bs4 import BeautifulSoup from playwright.sync_api import sync_playwright, expect import utils from spider.spiders.common.base_spider import BaseListToDetailsSpider from spider.spiders.common.models import Language from spider.spiders.common.models_v2 import TravelLocation, File, CommonModel, BigClass, TravelLocationComments, \ SocialUser # 创建临时目录 user_home = os.path.expanduser('~') temp_dir = os.path.join(user_home, 'temp_data_video') if not os.path.exists(temp_dir): os.makedirs(temp_dir) utils.log_print(f'--- 临时目录: {temp_dir}') class CrawlerType(Enum): SEARCH = 'keywords' # 按关键词搜索 USER = 'accounts' # 以用户ID为入口 POST = 'posts' # 以推文url为入口 TAG = 'tag' # 按话题搜索 # 旅游类爬虫 class TravelBookingSpider(BaseListToDetailsSpider): name = 'travel_booking' desc = '旅游-booking' net_name = 'Booking' net_name_zh = '缤客旅游网' net_url = 'www.booking.com' language = Language.EN nation_category = 1 channel_name = 'BookingHome' channel_url = 'https://www.booking.com/' channel_desc = 'Booking>Home' cookies = [ { "domain": ".booking.com", "expirationDate": 1781948826.462439, "hostOnly": False, "httpOnly": False, "name": "_ga", "path": "/", "secure": False, "session": False, "storeId": None, "value": "GA1.1.670792698.1747388754" }, { "domain": ".booking.com", "expirationDate": 1781948821.864516, "hostOnly": False, "httpOnly": True, "name": "bkng_sso_session", "path": "/", "secure": True, "session": False, "storeId": None, "value": "eyJib29raW5nX2dsb2JhbCI6W3sibG9naW5faGludCI6ImdwU1l1VUNTTGRVQ1FkaU9xendoTFUxQ1lpZWRHRnZKL3RDRHZMRW5MNVkifV19" }, { "domain": ".booking.com", "expirationDate": 1778924789.822941, "hostOnly": False, "httpOnly": True, "name": "explicit_language_preference", "path": "/", "secure": True, "session": False, "storeId": None, "value": "en-US" }, { "domain": ".booking.com", "expirationDate": 1747993626.465722, "hostOnly": False, "httpOnly": False, "name": "cgumid", "path": "/", "secure": False, "session": False, "storeId": None, "value": "auGwml9JNWVINSUyRk9vU3hOQkEwQUxRcTluS0MzcEl4RzNPaTd6WWV2T3JFUjhWWTR0Rk9ROXNvemoycGVHekE1a2RsQjIlMkI5ZEFBSGo2b21EQ290bEtVSEZNekElM0QlM0Q" }, { "domain": ".booking.com", "expirationDate": 1781948826.461974, "hostOnly": False, "httpOnly": False, "name": "_ga_A12345", "path": "/", "secure": False, "session": False, "storeId": None, "value": "GS2.1.s1747388755$o1$g1$t1747388826$j0$l0$h21144419" }, { "domain": ".booking.com", "expirationDate": 1747475225, "hostOnly": False, "httpOnly": False, "name": "_gid", "path": "/", "secure": False, "session": False, "storeId": None, "value": "GA1.2.862626268.1747388754" }, { "domain": ".booking.com", "hostOnly": False, "httpOnly": False, "name": "lastSeen", "path": "/", "secure": False, "session": True, "storeId": None, "value": "1747388832132" }, { "domain": ".booking.com", "expirationDate": 1755164756, "hostOnly": False, "httpOnly": False, "name": "_gcl_au", "path": "/", "secure": False, "session": False, "storeId": None, "value": "1.1.2071130618.1747388756" }, { "domain": ".booking.com", "expirationDate": 1747475226.054953, "hostOnly": False, "httpOnly": True, "name": "BJS", "path": "/", "secure": True, "session": False, "storeId": None, "value": "-" }, { "domain": ".booking.com", "expirationDate": 1747734433, "hostOnly": False, "httpOnly": False, "name": "aws-waf-token", "path": "/", "secure": True, "session": False, "storeId": None, "value": "d5d63b3b-15c3-4384-aa55-2ec4074f39d7:BgoAqYBDNfSNAAAA:RZpVYUAiH1SpGkcXjJvnks1bbNBLMnyAZ+O5nhlKf32YLyUTrvIjiqeLMtu2VDiOq+b7gSXwyd+sjmuqzg7o81R+0bGulTtIS/zvwDpFqKv+74gmHglAt2UHV4djycX3RMZext0dx3Gaqo+8cCFNo9XDdnsplPPTEofP6Pg2httP4IwSWGqrGsQKbRMNncHrdsMRVsdeyv5eEeIr8X/hb0/nwDUOoaUFB+GHrAyJW5m8rLOvD95JypK6U8+1k+a5nqk=" }, { "domain": ".booking.com", "expirationDate": 1781948831.725537, "hostOnly": False, "httpOnly": True, "name": "bkng", "path": "/", "secure": True, "session": False, "storeId": None, "value": "11UmFuZG9tSVYkc2RlIyh9Yaa29%2F3xUOLbA1O770KQL%2BxrCwOj88sIgVlty1hoH1Y9tsE4cWDItMaEyrluNWtAGPAxBpvwKnnVFNsjHa5Dq8yasE9h3WU2xLuvyXIheKUvllmsYrYsNl0ihnrwEs1g74ZT%2BDKqNqYPhDqFCozXtkirkfUo%2F1uXpwthK8Z4aekd14526YmrfNKgHNpo4lp1xgRd7YX%2FETqaKB0RRCSXBYYAfsJbPaHj%2FaItGZjK8jrA" }, { "domain": ".booking.com", "expirationDate": 1747392411.641568, "hostOnly": False, "httpOnly": True, "name": "dDfPWKZUFKYEYHCSCXHDINGYeQSAbFAFC", "path": "/", "secure": True, "session": False, "storeId": None, "value": "1" }, { "domain": ".booking.com", "expirationDate": 1781948821.405506, "hostOnly": False, "httpOnly": True, "name": "bkng_sso_auth", "path": "/", "secure": True, "session": False, "storeId": None, "value": "CAIQ0+WGHxqIAR63B8Iy3Z8m5ct/y0xyCXAQI0Mq3WPCCQkW7XacBBx4TE3zeG7KBy5WoXk90qW3iGI5zGF2T+QwHL2KpcZKKYinX4K3BmmF0TsmYP0ohVUv7LOqKiDVLccVkM/HuhtQ6EVRa46bE7fKkl5o/VpOhcYtUqICzYTJpPUSbti20nLRyz2PGA/p4jk=" }, { "domain": ".booking.com", "expirationDate": 1781552826, "hostOnly": False, "httpOnly": False, "name": "cto_bundle", "path": "/", "secure": False, "session": False, "storeId": None, "value": "IeFqRl92YkdIJTJGblklMkJIQ0JUbE5UQU5WbjNYRmtmUkF3U2NhaWg5dVVFUkpvR29YbVpDS3FrZ2c5ZXlPaVNsTyUyQjFCSXhqeUxQRDl4VEF5UE83SVNlWSUyQjlNVHR3dU11QjJhJTJCVEpSZWJiY3R3ZFU1b1piU1lkdGd5WXk3ejUlMkZ2UXFjTHZleQ" }, { "domain": ".booking.com", "expirationDate": 1778924753, "hostOnly": False, "httpOnly": False, "name": "cors_js", "path": "/", "secure": False, "session": False, "storeId": None, "value": "1" }, { "domain": ".booking.com", "expirationDate": 1747388885, "hostOnly": False, "httpOnly": False, "name": "_gat", "path": "/", "secure": False, "session": False, "storeId": None, "value": "1" }, { "domain": ".booking.com", "expirationDate": 1755164826, "hostOnly": False, "httpOnly": False, "name": "_uetmsclkid", "path": "/", "secure": False, "session": False, "storeId": None, "value": "_uet12e6d3213f3e1dbaa6e9bb063673b32c" }, { "domain": ".booking.com", "expirationDate": 1747475226, "hostOnly": False, "httpOnly": False, "name": "_uetsid", "path": "/", "secure": False, "session": False, "storeId": None, "value": "995d6f50323a11f0b5b333a90bf6b25a" }, { "domain": ".booking.com", "expirationDate": 1781084826, "hostOnly": False, "httpOnly": False, "name": "_uetvid", "path": "/", "secure": False, "session": False, "storeId": None, "value": "70dc65b0ecf811ef936d0718f3f6481f" }, { "domain": ".booking.com", "expirationDate": 1749980810, "hostOnly": False, "httpOnly": False, "name": "b", "path": "/", "secure": False, "session": False, "storeId": None, "value": "%7B%22countLang%22%3A1%7D" }, { "domain": ".booking.com", "hostOnly": False, "httpOnly": False, "name": "bkng_prue", "path": "/", "secure": False, "session": True, "storeId": None, "value": "1" }, { "domain": ".booking.com", "expirationDate": 1781948821.405553, "hostOnly": False, "httpOnly": True, "name": "bkng_sso_auth_1747388821", "path": "/", "secure": True, "session": False, "storeId": None, "value": "CAIQ0+WGHxqIAR63B8Iy3Z8m5ct/y0xyCXAQI0Mq3WPCCQkW7XacBBx4TE3zeG7KBy5WoXk90qW3iGI5zGF2T+QwHL2KpcZKKYinX4K3BmmF0TsmYP0ohVUv7LOqKiDVLccVkM/HuhtQ6EVRa46bE7fKkl5o/VpOhcYtUqICzYTJpPUSbti20nLRyz2PGA/p4jk=" }, { "domain": ".booking.com", "expirationDate": 1781948821.864388, "hostOnly": False, "httpOnly": True, "name": "bkng_sso_ses", "path": "/", "secure": True, "session": False, "storeId": None, "value": "eyJib29raW5nX2dsb2JhbCI6W3siaCI6ImdwU1l1VUNTTGRVQ1FkaU9xendoTFUxQ1lpZWRHRnZKL3RDRHZMRW5MNVkifV19" }, { "domain": ".booking.com", "expirationDate": 1778924801.118627, "hostOnly": False, "httpOnly": True, "name": "explicit_currency_preference", "path": "/", "secure": True, "session": False, "storeId": None, "value": "USD" }, { "domain": ".booking.com", "expirationDate": 1755164756.512236, "hostOnly": False, "httpOnly": False, "name": "FPAU", "path": "/", "secure": True, "session": False, "storeId": None, "value": "1.1.2071130618.1747388756" }, { "domain": ".booking.com", "expirationDate": 1781948827.512121, "hostOnly": False, "httpOnly": True, "name": "FPID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "FPID2.2.S8cTCYPTAdl4QEQ7ZTq2Cs1Fw5wcH6JKO68vYRc2I3E%3D.1747388754" }, { "domain": ".booking.com", "expirationDate": 1747460757.62276, "hostOnly": False, "httpOnly": False, "name": "FPLC", "path": "/", "secure": True, "session": False, "storeId": None, "value": "5nAO0cUZ8bU%2BRRKW8EAZ5BQn8epl%2BcKH%2F1FHUZWo57VypIANVGm6c1%2B%2FKRmSFghGIfGxsQ%2FErbMyBghn439izh%2B5eDvplogv5K6spmxTW63%2B%2FFMh92rj1UKUkVVDoQ%3D%3D" }, { "domain": ".www.booking.com", "expirationDate": 1762940825, "hostOnly": False, "httpOnly": False, "name": "OptanonConsent", "path": "/", "secure": True, "session": False, "storeId": None, "value": "implicitConsentCountry=nonGDPR&implicitConsentDate=1747388755646&isGpcEnabled=0&datestamp=Fri+May+16+2025+17%3A47%3A05+GMT%2B0800+(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)&version=202501.2.0&browserGpcFlag=0&isIABGlobal=False&hosts=&consentId=821ec330-9266-41b7-a0ed-e792262ae433&interactionCount=1&isAnonUser=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0002%3A1%2CC0004%3A1&AwaitingReconsent=False" }, { "domain": ".booking.com", "expirationDate": 1778924821.405454, "hostOnly": False, "httpOnly": True, "name": "pcm_consent", "path": "/", "secure": True, "session": False, "storeId": None, "value": "consentedAt%3D2025-05-16T09%3A47%3A01.237Z%26countryCode%3DSG%26expiresAt%3D2025-11-12T09%3A47%3A01.237Z%26implicit%3DFalse%26regulation%3Dnone%26legacyRegulation%3Dnone%26consentId%3D9f90d547-f966-4b95-83e1-90a57495cb00%26analytical%3DTrue%26marketing%3DTrue" }, { "domain": ".booking.com", "expirationDate": 1762940824.966122, "hostOnly": False, "httpOnly": True, "name": "pcm_personalization_disabled", "path": "/", "secure": True, "session": False, "storeId": None, "value": "0" } ] max_page = 3 # 最大爬取页数 processed_detail_urls = {} # 已处理的详情页链接 # 开始爬取 def start_requests(self): # 打开浏览器爬取 with sync_playwright() as p: headless = False if platform.system() == 'Windows' else True browser = p.chromium.launch(headless=headless) context = browser.new_context() context.add_cookies(self.cookies) page = context.new_page() # 爬取首页推荐专区 self.handle_list(page) utils.log_print(f'爬取结束') return [] def handle_list(self, page): page.goto('https://www.booking.com/searchresults.html?ss=%E8%8F%B2%E5%BE%8B%E5%AE%BE', timeout=1200000, wait_until='networkidle') # 随机生成本次滑动总次数（20-25次） total_scrolls = random.randint(20, 25) for i in range(total_scrolls): # 随机生成本次滑动距离（800-1500像素） scroll_distance = random.randint(800, 1500) page.evaluate(f"window.scrollBy(0, {scroll_distance});") # 随机延迟 page.wait_for_timeout(random.randint(1000, 2000)) page.wait_for_load_state('networkidle', timeout=300000) soup = BeautifulSoup(page.content(), 'html.parser') a_list = soup.find_all('a',attrs={'href':True,'data-testid':'title-link'}) for a in a_list: article_url = a['href'] if article_url not in self.processed_detail_urls: self.handle_detail(page, article_url) self.processed_detail_urls[article_url] = True def handle_detail(self, page, article_url): print(f'开始处理详情页: {article_url}') page.on('response', lambda response: self.process_comment_response(page, response, article_url)) page.goto(article_url, timeout=120000, wait_until='domcontentloaded') # 获取基本信息 name = page.locator('xpath=/html/body//main/div[1]/div[1]/div[1]/div[2]/div[4]/div[1]/div/div/h2').inner_text() content = page.locator('p[data-testid="property-description"]').inner_text() if not name or not content: return rating = '' try: rating = page.locator( 'xpath=/html/body//main/div[1]/div[1]/div[1]/div[4]/div/div[1]/div[1]/div/div[1]/a/div/div/div/div[2]').inner_text() results = re.findall(r'\d+', rating) if results: rating = results[0] except Exception as e: print(f'获取评分失败: {e}') rating_count = '' try: rating_count = page.locator( 'xpath=/html/body//main/div[1]/div[1]/div[1]/div[4]/div/div[1]/div[1]/div/div[1]/a/div/div/div/div[4]/div[2]').inner_text() results = re.findall(r'\d+', rating_count) if results: rating_count = results[0] except Exception as e: print(f'获取评分人数失败: {e}') tag = '' try: tag = page.locator('xpath=//nav[@id="breadcrumb"]').inner_text() except Exception as e: print(f'获取标签失败: {e}') address = '' try: address = page.locator( 'xpath=/html/body/div[4]/div/div[7]/main/div[1]/div[1]/div[1]/div[2]/div[4]/div[3]/div/div/span[1]/button/div/text()') except Exception as e: print(f'获取地址失败: {e}') img_elements = None try: img_elements = page.locator( 'xpath=/html/body//main/div[1]/div[1]/div[1]/div[3]/div/div/div/div/div[1]/div/div//img') except Exception as e: print(f'获取图片失败: {e}') booking_type = '' if '/hotel/' in article_url: booking_type = '酒店' else: booking_type = '景点' travel_location = TravelLocation(detail_url=article_url, name=name, description=content, tag=tag, rating=rating, rating_count=rating_count, address=address, type=booking_type) files = [] img_count = img_elements.count() if img_elements else 0 for img_num in range(img_count): img_element = img_elements.nth(img_num) img_url = img_element.get_attribute('src') files.append( File(title='图片', file_url=img_url, file_type='img', file_ext='.jpg',src_url=article_url, src_id=travel_location.id)) # 保存基本信息 entities = { 'travel_locations': [travel_location], 'files': files } product = CommonModel(spider=self, big_class=BigClass.TRAVEL, entities=entities) product.save_file() for img_url in files[:5]: product.save_img_file(img_url.file_url) # # 获取评论按钮点击 # page.locator('button[data-testid="fr-read-all-reviews"]').click() # time.sleep(random.randint(3, 5)) # 评论翻页，一直点击下一页按钮知道灰显为止 # 暂时只翻20页，todo，后续可以增加翻页逻辑 # next_page = page.locator('button[class="a83ed08757 c21c56c305 f38b6daa18 d691166b09 ab98298258 bb803d8689 a16ddf9c57"]') # next_page = page.locator('button[aria-label="下一页"]') # if next_page.count() == 1: # while not next_page.is_disabled(): # next_page.click() # time.sleep(random.randint(1, 3)) # roll_count = 0 # if not next_page.is_disabled() and roll_count < 20: # roll_count += 1 # next_page.click() # time.sleep(random.randint(1, 3)) # 处理评论响应，todo，demo，后续需要重构，暂时只抓取一页评论 def process_comment_response(self, page, response, article_url): try: url = response.url if "https://www.booking.com/dml/graphql" in url or 'https://www.booking.cn/dml/graphql' in url: status_code = response.status if status_code == 200: api_name = None if response.request: post_data_json = response.request.headers api_name = None try: api_name = post_data_json.get('x-apollo-operation-name') except Exception as e: print(f'获取api_name失败: {e}') if api_name and api_name == 'ReviewList': json = response.json() comment_list = json.get('data', {}).get('reviewListFrontend', []).get('reviewCard', []) for comment in comment_list: username = comment.get('guestDetails', {}).get('username') countryName = comment.get('guestDetails', {}).get('countryName') countryCode = comment.get('guestDetails', {}).get('countryCode') avatar_url = comment.get('guestDetails', {}).get('avatarUrl') reviewScore = comment.get('reviewScore') reviewedDate = comment.get('reviewedDate') title = comment.get('textDetails', {}).get('title', '') positiveText = comment.get('textDetails', {}).get('positiveText', '') negativeText = comment.get('textDetails', {}).get('negativeText', '') final_comment = title or positiveText or negativeText print( f'username: {username}, countryName: {countryName}, countryCode:{countryCode},reviewScore: {reviewScore}, title: {title}, positiveText: {positiveText}, negativeText: {negativeText}') comment = TravelLocationComments(post_url=article_url, post_id=utils.str_to_md5(article_url), username=username, user_location=countryName, rating=reviewScore, pub_time=reviewedDate, comment=final_comment) user = SocialUser(user_url='https://www.booking.cn/user/' + username, username=username, profile_image_url=avatar_url, nation=countryName) if not comment.id or not user.id: print(f'旅游类别，保存评论失败: comment:{comment}, user:{user}') continue comment_model = CommonModel(spider=self, big_class=BigClass.TRAVEL, entities={'travel_comments': [comment], 'travel_users': [user]}) comment_model.save_file() print(f'保存评论成功: {comment.id}') page.close() except: print(f'处理评论响应失败: {traceback.format_exc()}')

代码

import os
import platform
import random
import re
import time
import traceback
from enum import Enum

from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright, expect

import utils
from spider.spiders.common.base_spider import BaseListToDetailsSpider
from spider.spiders.common.models import Language
from spider.spiders.common.models_v2 import TravelLocation, File, CommonModel, BigClass, TravelLocationComments, \
    SocialUser

# 创建临时目录
user_home = os.path.expanduser('~')
temp_dir = os.path.join(user_home, 'temp_data_video')
if not os.path.exists(temp_dir):
    os.makedirs(temp_dir)
utils.log_print(f'--- 临时目录: {temp_dir}')

class CrawlerType(Enum):
    SEARCH = 'keywords'  # 按关键词搜索
    USER = 'accounts'  # 以用户ID为入口
    POST = 'posts'  # 以推文url为入口
    TAG = 'tag'  # 按话题搜索

# 旅游类爬虫
class TravelBookingSpider(BaseListToDetailsSpider):
    name = 'travel_booking'
    desc = '旅游-booking'
    net_name = 'Booking'
    net_name_zh = '缤客旅游网'
    net_url = 'www.booking.com'
    language = Language.EN
    nation_category = 1
    channel_name = 'BookingHome'
    channel_url = 'https://www.booking.com/'
    channel_desc = 'Booking>Home'

cookies = [
        {
            "domain": ".booking.com",
            "expirationDate": 1781948826.462439,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_ga",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "GA1.1.670792698.1747388754"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948821.864516,
            "hostOnly": False,
            "httpOnly": True,
            "name": "bkng_sso_session",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "eyJib29raW5nX2dsb2JhbCI6W3sibG9naW5faGludCI6ImdwU1l1VUNTTGRVQ1FkaU9xendoTFUxQ1lpZWRHRnZKL3RDRHZMRW5MNVkifV19"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1778924789.822941,
            "hostOnly": False,
            "httpOnly": True,
            "name": "explicit_language_preference",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "en-US"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747993626.465722,
            "hostOnly": False,
            "httpOnly": False,
            "name": "cgumid",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "auGwml9JNWVINSUyRk9vU3hOQkEwQUxRcTluS0MzcEl4RzNPaTd6WWV2T3JFUjhWWTR0Rk9ROXNvemoycGVHekE1a2RsQjIlMkI5ZEFBSGo2b21EQ290bEtVSEZNekElM0QlM0Q"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948826.461974,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_ga_A12345",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "GS2.1.s1747388755$o1$g1$t1747388826$j0$l0$h21144419"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747475225,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_gid",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "GA1.2.862626268.1747388754"
        },
        {
            "domain": ".booking.com",
            "hostOnly": False,
            "httpOnly": False,
            "name": "lastSeen",
            "path": "/",
            "secure": False,
            "session": True,
            "storeId": None,
            "value": "1747388832132"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1755164756,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_gcl_au",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "1.1.2071130618.1747388756"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747475226.054953,
            "hostOnly": False,
            "httpOnly": True,
            "name": "BJS",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "-"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747734433,
            "hostOnly": False,
            "httpOnly": False,
            "name": "aws-waf-token",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "d5d63b3b-15c3-4384-aa55-2ec4074f39d7:BgoAqYBDNfSNAAAA:RZpVYUAiH1SpGkcXjJvnks1bbNBLMnyAZ+O5nhlKf32YLyUTrvIjiqeLMtu2VDiOq+b7gSXwyd+sjmuqzg7o81R+0bGulTtIS/zvwDpFqKv+74gmHglAt2UHV4djycX3RMZext0dx3Gaqo+8cCFNo9XDdnsplPPTEofP6Pg2httP4IwSWGqrGsQKbRMNncHrdsMRVsdeyv5eEeIr8X/hb0/nwDUOoaUFB+GHrAyJW5m8rLOvD95JypK6U8+1k+a5nqk="
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948831.725537,
            "hostOnly": False,
            "httpOnly": True,
            "name": "bkng",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "11UmFuZG9tSVYkc2RlIyh9Yaa29%2F3xUOLbA1O770KQL%2BxrCwOj88sIgVlty1hoH1Y9tsE4cWDItMaEyrluNWtAGPAxBpvwKnnVFNsjHa5Dq8yasE9h3WU2xLuvyXIheKUvllmsYrYsNl0ihnrwEs1g74ZT%2BDKqNqYPhDqFCozXtkirkfUo%2F1uXpwthK8Z4aekd14526YmrfNKgHNpo4lp1xgRd7YX%2FETqaKB0RRCSXBYYAfsJbPaHj%2FaItGZjK8jrA"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747392411.641568,
            "hostOnly": False,
            "httpOnly": True,
            "name": "dDfPWKZUFKYEYHCSCXHDINGYeQSAbFAFC",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "1"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948821.405506,
            "hostOnly": False,
            "httpOnly": True,
            "name": "bkng_sso_auth",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "CAIQ0+WGHxqIAR63B8Iy3Z8m5ct/y0xyCXAQI0Mq3WPCCQkW7XacBBx4TE3zeG7KBy5WoXk90qW3iGI5zGF2T+QwHL2KpcZKKYinX4K3BmmF0TsmYP0ohVUv7LOqKiDVLccVkM/HuhtQ6EVRa46bE7fKkl5o/VpOhcYtUqICzYTJpPUSbti20nLRyz2PGA/p4jk="
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781552826,
            "hostOnly": False,
            "httpOnly": False,
            "name": "cto_bundle",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "IeFqRl92YkdIJTJGblklMkJIQ0JUbE5UQU5WbjNYRmtmUkF3U2NhaWg5dVVFUkpvR29YbVpDS3FrZ2c5ZXlPaVNsTyUyQjFCSXhqeUxQRDl4VEF5UE83SVNlWSUyQjlNVHR3dU11QjJhJTJCVEpSZWJiY3R3ZFU1b1piU1lkdGd5WXk3ejUlMkZ2UXFjTHZleQ"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1778924753,
            "hostOnly": False,
            "httpOnly": False,
            "name": "cors_js",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "1"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747388885,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_gat",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "1"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1755164826,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_uetmsclkid",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "_uet12e6d3213f3e1dbaa6e9bb063673b32c"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747475226,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_uetsid",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "995d6f50323a11f0b5b333a90bf6b25a"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781084826,
            "hostOnly": False,
            "httpOnly": False,
            "name": "_uetvid",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "70dc65b0ecf811ef936d0718f3f6481f"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1749980810,
            "hostOnly": False,
            "httpOnly": False,
            "name": "b",
            "path": "/",
            "secure": False,
            "session": False,
            "storeId": None,
            "value": "%7B%22countLang%22%3A1%7D"
        },
        {
            "domain": ".booking.com",
            "hostOnly": False,
            "httpOnly": False,
            "name": "bkng_prue",
            "path": "/",
            "secure": False,
            "session": True,
            "storeId": None,
            "value": "1"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948821.405553,
            "hostOnly": False,
            "httpOnly": True,
            "name": "bkng_sso_auth_1747388821",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "CAIQ0+WGHxqIAR63B8Iy3Z8m5ct/y0xyCXAQI0Mq3WPCCQkW7XacBBx4TE3zeG7KBy5WoXk90qW3iGI5zGF2T+QwHL2KpcZKKYinX4K3BmmF0TsmYP0ohVUv7LOqKiDVLccVkM/HuhtQ6EVRa46bE7fKkl5o/VpOhcYtUqICzYTJpPUSbti20nLRyz2PGA/p4jk="
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948821.864388,
            "hostOnly": False,
            "httpOnly": True,
            "name": "bkng_sso_ses",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "eyJib29raW5nX2dsb2JhbCI6W3siaCI6ImdwU1l1VUNTTGRVQ1FkaU9xendoTFUxQ1lpZWRHRnZKL3RDRHZMRW5MNVkifV19"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1778924801.118627,
            "hostOnly": False,
            "httpOnly": True,
            "name": "explicit_currency_preference",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "USD"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1755164756.512236,
            "hostOnly": False,
            "httpOnly": False,
            "name": "FPAU",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "1.1.2071130618.1747388756"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1781948827.512121,
            "hostOnly": False,
            "httpOnly": True,
            "name": "FPID",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "FPID2.2.S8cTCYPTAdl4QEQ7ZTq2Cs1Fw5wcH6JKO68vYRc2I3E%3D.1747388754"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1747460757.62276,
            "hostOnly": False,
            "httpOnly": False,
            "name": "FPLC",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "5nAO0cUZ8bU%2BRRKW8EAZ5BQn8epl%2BcKH%2F1FHUZWo57VypIANVGm6c1%2B%2FKRmSFghGIfGxsQ%2FErbMyBghn439izh%2B5eDvplogv5K6spmxTW63%2B%2FFMh92rj1UKUkVVDoQ%3D%3D"
        },
        {
            "domain": ".www.booking.com",
            "expirationDate": 1762940825,
            "hostOnly": False,
            "httpOnly": False,
            "name": "OptanonConsent",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "implicitConsentCountry=nonGDPR&implicitConsentDate=1747388755646&isGpcEnabled=0&datestamp=Fri+May+16+2025+17%3A47%3A05+GMT%2B0800+(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)&version=202501.2.0&browserGpcFlag=0&isIABGlobal=False&hosts=&consentId=821ec330-9266-41b7-a0ed-e792262ae433&interactionCount=1&isAnonUser=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0002%3A1%2CC0004%3A1&AwaitingReconsent=False"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1778924821.405454,
            "hostOnly": False,
            "httpOnly": True,
            "name": "pcm_consent",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "consentedAt%3D2025-05-16T09%3A47%3A01.237Z%26countryCode%3DSG%26expiresAt%3D2025-11-12T09%3A47%3A01.237Z%26implicit%3DFalse%26regulation%3Dnone%26legacyRegulation%3Dnone%26consentId%3D9f90d547-f966-4b95-83e1-90a57495cb00%26analytical%3DTrue%26marketing%3DTrue"
        },
        {
            "domain": ".booking.com",
            "expirationDate": 1762940824.966122,
            "hostOnly": False,
            "httpOnly": True,
            "name": "pcm_personalization_disabled",
            "path": "/",
            "secure": True,
            "session": False,
            "storeId": None,
            "value": "0"
        }
    ]

max_page = 3  # 最大爬取页数
    processed_detail_urls = {}  # 已处理的详情页链接

# 开始爬取
    def start_requests(self):
        # 打开浏览器爬取
        with sync_playwright() as p:
            headless = False if platform.system() == 'Windows' else True
            browser = p.chromium.launch(headless=headless)
            context = browser.new_context()
            context.add_cookies(self.cookies)
            page = context.new_page()

# 爬取首页推荐专区
            self.handle_list(page)

utils.log_print(f'爬取结束')
        return []

def handle_list(self, page):
        page.goto('https://www.booking.com/searchresults.html?ss=%E8%8F%B2%E5%BE%8B%E5%AE%BE', timeout=1200000,
                  wait_until='networkidle')

# 随机生成本次滑动总次数（20-25次）
        total_scrolls = random.randint(20, 25)
        for i in range(total_scrolls):
            # 随机生成本次滑动距离（800-1500像素）
            scroll_distance = random.randint(800, 1500)
            page.evaluate(f"window.scrollBy(0, {scroll_distance});")
            # 随机延迟
            page.wait_for_timeout(random.randint(1000, 2000))
        page.wait_for_load_state('networkidle', timeout=300000)
        soup = BeautifulSoup(page.content(), 'html.parser')
        a_list = soup.find_all('a',attrs={'href':True,'data-testid':'title-link'})
        for a in a_list:
            article_url = a['href']
            if article_url not in self.processed_detail_urls:
                self.handle_detail(page, article_url)
                self.processed_detail_urls[article_url] = True

def handle_detail(self, page, article_url):
        print(f'开始处理详情页: {article_url}')
        page.on('response',
                lambda response: self.process_comment_response(page, response, article_url))

page.goto(article_url, timeout=120000, wait_until='domcontentloaded')

# 获取基本信息
        name = page.locator('xpath=/html/body//main/div[1]/div[1]/div[1]/div[2]/div[4]/div[1]/div/div/h2').inner_text()
        content = page.locator('p[data-testid="property-description"]').inner_text()

if not name or not content:
            return

rating = ''
        try:
            rating = page.locator(
                'xpath=/html/body//main/div[1]/div[1]/div[1]/div[4]/div/div[1]/div[1]/div/div[1]/a/div/div/div/div[2]').inner_text()
            results = re.findall(r'\d+', rating)
            if results:
                rating = results[0]
        except Exception as e:
            print(f'获取评分失败: {e}')
        rating_count = ''
        try:
            rating_count = page.locator(
                'xpath=/html/body//main/div[1]/div[1]/div[1]/div[4]/div/div[1]/div[1]/div/div[1]/a/div/div/div/div[4]/div[2]').inner_text()
            results = re.findall(r'\d+', rating_count)
            if results:
                rating_count = results[0]
        except Exception as e:
            print(f'获取评分人数失败: {e}')
        tag = ''
        try:
            tag = page.locator('xpath=//nav[@id="breadcrumb"]').inner_text()
        except Exception as e:
            print(f'获取标签失败: {e}')
        address = ''
        try:
            address = page.locator(
                'xpath=/html/body/div[4]/div/div[7]/main/div[1]/div[1]/div[1]/div[2]/div[4]/div[3]/div/div/span[1]/button/div/text()')
        except Exception as e:
            print(f'获取地址失败: {e}')
        img_elements = None
        try:
            img_elements = page.locator(
                'xpath=/html/body//main/div[1]/div[1]/div[1]/div[3]/div/div/div/div/div[1]/div/div//img')
        except Exception as e:
            print(f'获取图片失败: {e}')

booking_type = ''
        if '/hotel/' in article_url:
            booking_type = '酒店'
        else:
            booking_type = '景点'

travel_location = TravelLocation(detail_url=article_url,
                                         name=name,
                                         description=content,
                                         tag=tag,
                                         rating=rating,
                                         rating_count=rating_count,
                                         address=address,
                                         type=booking_type)

files = []
        img_count = img_elements.count() if img_elements else 0
        for img_num in range(img_count):
            img_element = img_elements.nth(img_num)
            img_url = img_element.get_attribute('src')
            files.append(
                File(title='图片', file_url=img_url, file_type='img', file_ext='.jpg',src_url=article_url, src_id=travel_location.id))

# 保存基本信息
        entities = {
            'travel_locations': [travel_location],
            'files': files
        }

product = CommonModel(spider=self, big_class=BigClass.TRAVEL, entities=entities)
        product.save_file()
        for img_url in files[:5]:
            product.save_img_file(img_url.file_url)

# # 获取评论按钮点击
        # page.locator('button[data-testid="fr-read-all-reviews"]').click()
        # time.sleep(random.randint(3, 5))

# 评论翻页，一直点击下一页按钮知道灰显为止
        # 暂时只翻20页，todo，后续可以增加翻页逻辑
        # next_page = page.locator('button[class="a83ed08757 c21c56c305 f38b6daa18 d691166b09 ab98298258 bb803d8689 a16ddf9c57"]')
        # next_page = page.locator('button[aria-label="下一页"]')
        # if next_page.count() == 1:
        #     while not next_page.is_disabled():
        #         next_page.click()
        #         time.sleep(random.randint(1, 3))

# roll_count = 0
        # if not next_page.is_disabled() and roll_count < 20:
        #     roll_count += 1
        #     next_page.click()
        #     time.sleep(random.randint(1, 3))

# 处理评论响应，todo，demo，后续需要重构，暂时只抓取一页评论
    def process_comment_response(self, page, response, article_url):
        try:
            url = response.url
            if "https://www.booking.com/dml/graphql" in url or 'https://www.booking.cn/dml/graphql' in url:
                status_code = response.status
                if status_code == 200:
                    api_name = None
                    if response.request:
                        post_data_json = response.request.headers
                        api_name = None
                        try:
                            api_name = post_data_json.get('x-apollo-operation-name')
                        except Exception as e:
                            print(f'获取api_name失败: {e}')
                    if api_name and api_name == 'ReviewList':
                        json = response.json()
                        comment_list = json.get('data', {}).get('reviewListFrontend', []).get('reviewCard', [])
                        for comment in comment_list:
                            username = comment.get('guestDetails', {}).get('username')
                            countryName = comment.get('guestDetails', {}).get('countryName')
                            countryCode = comment.get('guestDetails', {}).get('countryCode')
                            avatar_url = comment.get('guestDetails', {}).get('avatarUrl')

reviewScore = comment.get('reviewScore')
                            reviewedDate = comment.get('reviewedDate')
                            title = comment.get('textDetails', {}).get('title', '')
                            positiveText = comment.get('textDetails', {}).get('positiveText', '')
                            negativeText = comment.get('textDetails', {}).get('negativeText', '')
                            final_comment = title or positiveText or negativeText
                            print(
                                f'username: {username}, countryName: {countryName}, countryCode:{countryCode},reviewScore: {reviewScore}, title: {title}, positiveText: {positiveText}, negativeText: {negativeText}')

comment = TravelLocationComments(post_url=article_url,
                                                             post_id=utils.str_to_md5(article_url), username=username,
                                                             user_location=countryName, rating=reviewScore,
                                                             pub_time=reviewedDate, comment=final_comment)

user = SocialUser(user_url='https://www.booking.cn/user/' + username, username=username,
                                              profile_image_url=avatar_url, nation=countryName)

if not comment.id or not user.id:
                                print(f'旅游类别，保存评论失败: comment:{comment}, user:{user}')
                                continue

comment_model = CommonModel(spider=self, big_class=BigClass.TRAVEL,
                                                        entities={'travel_comments': [comment], 'travel_users': [user]})
                            comment_model.save_file()
                            print(f'保存评论成功: {comment.id}')

page.close()
        except:
            print(f'处理评论响应失败: {traceback.format_exc()}')