编辑采集器       终止采集器

属性	值	属性	值
名称	us_youtube_multimedia	创建时间	2025-04-12 09:33:57
标题	美国-油管视频网站	更新时间	-
状态	running	开始时间	2025-06-27 12:08:57
停止类型		结束时间	-
信号		运行时长	248.1小时
进程ID	4044581	进程名称	Process-5
作业ID	4e2ecc6c-340f-4695-b655-d0e2d147162b	作业目录	records/jobs/4e2ecc6c-340f-4695-b655-d0e2d147162b
日志文件	records/logs/4e2ecc6c-340f-4695-b655-d0e2d147162b.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_youtube_multimedia_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
11494214-2859-47c7-bd80-019bd6f635e6	2025-06-20 18:07:22	2025-06-27 20:08:52	170.0小时	1568267	Process-4	查看日志	删除记录
12c20993-fa65-43b4-9e01-f0bf1e2bebb9	2025-06-18 18:51:16	2025-06-20 18:07:17	47.3小时	3261986	Process-7	查看日志	删除记录
24e4b1d7-973d-4f33-9823-401f39505413	2025-06-18 18:46:42	2025-06-18 18:51:13	4.5分钟	3243305	Process-4	查看日志	删除记录
4ddd0d77-b87f-4384-ad7f-6161e6d54fd1	2025-05-28 17:16:34	2025-06-05 14:31:53	189.3小时	2440875	Process-29	查看日志	删除记录
992c3a2e-965a-4709-b1cd-bb810709e81e	2025-05-24 19:07:13	2025-05-26 14:18:33	43.2小时	590103	Process-31	查看日志	删除记录
30eaf7ef-ff75-4961-b8cf-dc05bab5b171	2025-05-22 09:14:40	2025-05-22 09:19:54	5.2分钟	3293432	Process-32	查看日志	删除记录
ccb9bf2a-1ca9-48cd-96ad-1672da688e85	2025-05-21 06:23:12	-	1149.9小时	3074638	Process-24	查看日志	删除记录
41f64370-8a60-460d-8b34-a522028dfd1a	2025-05-19 16:53:51	-	1187.3小时	3859469	Process-23	查看日志	删除记录
98046b92-7f79-493b-b333-d20e0046f49b	2025-05-16 21:08:10	2025-05-19 16:53:47	67.8小时	2563771	Process-11	查看日志	删除记录
7dec3946-19cf-46e6-960d-febabf288ef4	2025-05-14 17:06:00	2025-05-16 18:45:35	49.7小时	445088	Process-24	查看日志	删除记录
d3895953-27c1-4efd-9b3b-7ecd62bbf775	2025-05-14 16:16:02	2025-05-14 17:04:12	48.2分钟	374918	Process-21	查看日志	删除记录
59166297-d2bb-4b46-9d8b-ff64a05e3be0	2025-04-12 19:24:11	2025-05-14 14:52:14	763.5小时	451734	Process-4	查看日志	删除记录
删除最后 选择条数 条历史作业 确定

代码

import os import time import random from bs4 import BeautifulSoup import platform import traceback from playwright.sync_api import sync_playwright, expect, Response import dao import utils from spider.spiders.common.models_v2 import SocialUser, File, CommonModel, BigClass, VideoComment, VideoInfo from spider.spiders.common.base_spider import BaseListToDetailsSpider # 创建临时目录 user_home = os.path.expanduser('~') temp_dir = os.path.join(user_home, 'temp_data_video') if not os.path.exists(temp_dir): os.makedirs(temp_dir) utils.log_print(f'--- 临时目录: {temp_dir}') import json from datetime import datetime class UsYoutubeMultimediaSpider(BaseListToDetailsSpider): name = "us_youtube_multimedia" desc = "美国-油管视频网站" net_name = 'youtube' net_url = 'www.youtube.com' net_name_zh = '油管' net_nation = '美国' net_language = 'en' channel_name = 'daily recommendation' nation_category = 1 channel_url = 'https://www.youtube.com' channel_desc = 'youtube' timeout_count = 0 # 读取队列最大超时次数 max_timeout_count = 6 all_final_urls = [] cookies_list = [ { "domain": ".youtube.com", "expirationDate": 1755512105.361403, "hostOnly": False, "httpOnly": True, "name": "VISITOR_PRIVACY_METADATA", "path": "/", "secure": True, "session": False, "storeId": None, "value": "CgJTRxIEGgAgWA%3D%3D" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316768, "hostOnly": False, "httpOnly": True, "name": "__Secure-3PSID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "g.a000twhici2G71xt8StEL4bSj4Dnm8BMhqVqYoQ04h2MMONWAyXLo9Pbz-ALQgJ9pAy7mnQmGQACgYKAc4SARISFQHGX2MixMozF9RVDmEX_2S52E1oghoVAUF8yKqOfQkQ2nd9IHypfBYrwIA00076" }, { "domain": ".youtube.com", "expirationDate": 1772013874.553871, "hostOnly": False, "httpOnly": False, "name": "SIDCC", "path": "/", "secure": False, "session": False, "storeId": None, "value": "AKEyXzU21ZadHIpoNEUUaftXBYIyN_lzCabh8SIqYpIjCkoCai7HCTJEWbR_1HPRsUuyxLKOhhc" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316722, "hostOnly": False, "httpOnly": False, "name": "SID", "path": "/", "secure": False, "session": False, "storeId": None, "value": "g.a000twhici2G71xt8StEL4bSj4Dnm8BMhqVqYoQ04h2MMONWAyXLTePNtiilJqQ2Rv9txhvKgQACgYKAf8SARISFQHGX2MiJgByQ9SHc_1qTfTT-mJ5oxoVAUF8yKpEFPfoIrq3vMXslftgoKRt0076" }, { "domain": ".youtube.com", "expirationDate": 1772013874.553739, "hostOnly": False, "httpOnly": True, "name": "__Secure-1PSIDTS", "path": "/", "secure": True, "session": False, "storeId": None, "value": "sidts-CjIBEJ3XVwRzXgY5t5TQ_t4ZqWHZHbtnMe0l02UcztCOtkT_M_edFZ6ISDshlY1MKLZ4xRAA" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316526, "hostOnly": False, "httpOnly": False, "name": "SAPISID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "lNbf6yFa_j5gFRpU/Ay-qMvCojftuQT4R0" }, { "domain": ".youtube.com", "expirationDate": 1772013874.553905, "hostOnly": False, "httpOnly": True, "name": "__Secure-1PSIDCC", "path": "/", "secure": True, "session": False, "storeId": None, "value": "AKEyXzUfARzDaHENKoB4rZg4tQaRDCbKP3p9oYOJ6_IhqeSkpyRtwkoT6iib1ppeslQG0pLnmRE" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316477, "hostOnly": False, "httpOnly": True, "name": "SSID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "AgKsW1AnV7WUpJw7r" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316548, "hostOnly": False, "httpOnly": False, "name": "__Secure-1PAPISID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "lNbf6yFa_j5gFRpU/Ay-qMvCojftuQT4R0" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316745, "hostOnly": False, "httpOnly": True, "name": "__Secure-1PSID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "g.a000twhici2G71xt8StEL4bSj4Dnm8BMhqVqYoQ04h2MMONWAyXLeBbbq98_T2Lyvj-R8oghvwACgYKAXMSARISFQHGX2MiJrmm-nuj1cwF_NS2LJq_nBoVAUF8yKq5ELDzVV7lr2o61mLSp2o90076" }, { "domain": ".youtube.com", "expirationDate": 1774489906.31657, "hostOnly": False, "httpOnly": False, "name": "__Secure-3PAPISID", "path": "/", "secure": True, "session": False, "storeId": None, "value": "lNbf6yFa_j5gFRpU/Ay-qMvCojftuQT4R0" }, { "domain": ".youtube.com", "expirationDate": 1772013874.553941, "hostOnly": False, "httpOnly": True, "name": "__Secure-3PSIDCC", "path": "/", "secure": True, "session": False, "storeId": None, "value": "AKEyXzUKBVzWBiyRofUHXb-PSEUUemjCmDcSI-wy1b2EaPq8xhzdzuR1vT3ahsxsTkUOrvS12EQ" }, { "domain": ".youtube.com", "expirationDate": 1772013874.553835, "hostOnly": False, "httpOnly": True, "name": "__Secure-3PSIDTS", "path": "/", "secure": True, "session": False, "storeId": None, "value": "sidts-CjIBEJ3XVwRzXgY5t5TQ_t4ZqWHZHbtnMe0l02UcztCOtkT_M_edFZ6ISDshlY1MKLZ4xRAA" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316502, "hostOnly": False, "httpOnly": False, "name": "APISID", "path": "/", "secure": False, "session": False, "storeId": None, "value": "g4CY7Rc-BN4RNRLM/A89twSwWyLr83b7Ax" }, { "domain": ".youtube.com", "expirationDate": 1774489906.316351, "hostOnly": False, "httpOnly": True, "name": "HSID", "path": "/", "secure": False, "session": False, "storeId": None, "value": "A8vQauEYC-hl7d8kS" }, { "domain": ".youtube.com", "expirationDate": 1743261373.55498, "hostOnly": False, "httpOnly": True, "name": "LOGIN_INFO", "path": "/", "secure": True, "session": False, "storeId": None, "value": "AFmmF2swRQIhAPdgVKqRKLfQbGsLJ8yFIfkZRpfqoGLhEl4n0GIyNT3qAiAh_nk35NcJsCQ7_nq-u7yj3ePL1poQGpHhZOoSBcsMLQ:QUQ3MjNmd2k3djFVRzIzdzBYdllDZmppZXU0bE1sN1JEVHJoSFhGTTY4bjhYUEEwZ2ljNEpfekFUNDRHSm50NEUyZnZyNE5iS2NZN3ZfREpfblBEOVBmczRtcWV0Zk1NZmlFUWdyelZYVjN6WFRIZnVwTjFVN1pMbWVMWk9zQXp0bTdqTnRUYjk5dVZibWhzVGhQSEZmSVFZZXAxb0hXa1FB" }, { "domain": ".youtube.com", "expirationDate": 1775036208.614868, "hostOnly": False, "httpOnly": False, "name": "PREF", "path": "/", "secure": True, "session": False, "storeId": None, "value": "f7=4100&tz=Asia.Shanghai&f4=4000000&f6=40000000" }, { "domain": ".youtube.com", "expirationDate": 1755512105.361165, "hostOnly": False, "httpOnly": True, "name": "VISITOR_INFO1_LIVE", "path": "/", "secure": True, "session": False, "storeId": None, "value": "ubJ7XpWgIuE" } ] def start_requests(self): """合并逻辑：先爬取链接存入Redis，再消费Redis中的任务，共用一个浏览器""" # 初始化变量 self.browser = None self.context = None self.page = None try: # 1. 启动浏览器（复用实例） with sync_playwright() as p: # Windows显示浏览器，Linux无头模式 headless = False if platform.system() == 'Windows' else True self.browser = p.chromium.launch(headless=headless) self.context = self.browser.new_context() self.page = self.context.new_page() # 2. 第一步：爬取首页链接，存入Redis utils.log_print("开始爬取首页链接并存入Redis...") self.handle_search(page=self.page) # 调用原搜索逻辑 utils.log_print("首页链接爬取完成，开始处理任务...") # 3. 第二步：消费Redis中的任务（复用同一个浏览器） self.handle_details_reuse_browser() # 新方法：复用浏览器处理任务 except Exception as e: utils.log_print(f"整体流程出错：{traceback.format_exc()}") finally: # 确保资源释放 if self.page: self.page.close() if self.context: self.context.close() if self.browser: self.browser.close() return [] def handle_details_reuse_browser(self): """复用浏览器处理Redis中的任务（替代原handle_details方法）""" # 直接使用start_requests中创建的page，无需重新启动浏览器 page = self.page utils.log_print("开始消费Redis中的任务...") while True: time.sleep(random.randint(3, 5)) # 避免频繁轮询 try: # 从Redis获取任务 post_dict = dao.get_spider_work_queue_wait(self.name, 30) if not post_dict: self.timeout_count += 1 utils.log_print(f"未获取到任务，当前超时次数：{self.timeout_count}") # 超过最大超时次数则退出（可根据需求调整） if self.timeout_count > self.max_timeout_count: utils.log_print(f"超过{self.max_timeout_count}次未获取任务，退出消费") break continue # 重置超时计数（获取到任务） self.timeout_count = 0 post_dict = json.loads(post_dict) publish_url = post_dict.get("post_url") utils.log_print(f"开始处理任务：{publish_url}") # 调用原handle_page处理任务（复用当前page） self.handle_page(page=page) except Exception as e: utils.log_print(f"处理任务 {publish_url} 出错：{traceback.format_exc()}") # 任务失败时放回队列（保持原逻辑） if publish_url: dao.put_spider_work_queue(self.name, json.dumps(post_dict)) def handle_page(self, page): max_retries = 3 retry_count = 0 while retry_count < max_retries: try: files = [] post_dict = dao.get_spider_work_queue_wait(self.name) if not post_dict: self.timeout_count += 1 if self.timeout_count > self.max_timeout_count: utils.log_print(f"超过{self.max_timeout_count}次未获取到待处理用户，退出") break post_dict = json.loads(post_dict) url = post_dict.get("post_url") duration = post_dict.get("duration", '') title = post_dict.get("title", '') video_img_link = post_dict.get("video_img_link", '') tag = post_dict.get("tag", '') author = post_dict.get("author", '') pub_time = post_dict.get("pub_time", '') view_count = post_dict.get("view_count", '') time_load = 60000 page.goto(url, wait_until="load", timeout=time_load) # 监听响应事件 page.on("response", lambda response: self.handle_response(response,url)) #查找评论的次数，根据评论数进行动态翻页 find_comment_count = 0 while True: # 检查元素是否可见 if page.locator('yt-formatted-string.count-text.style-scope.ytd-comments-header-renderer').is_visible(): break # 向下滚动一小段距离 page.evaluate(f"window.scrollBy(0,{random.randint(500, 600)})") # 滚动 1500-2000 像素 # 等待一小段时间，避免过度滚动 page.wait_for_timeout(random.randint(2000, 4000)) page.wait_for_load_state("networkidle") find_comment_count += 1 if find_comment_count > 3: utils.log_print(f"评论数获取失败，跳过当前视频") break soup = BeautifulSoup(page.content(), 'lxml') page.wait_for_timeout(random.randint(3000, 5000)) try: #喜欢 like_count = '' like_button = soup.find('button',attrs={'class':'yt-spec-button-shape-next yt-spec-button-shape-next--tonal yt-spec-button-shape-next--mono yt-spec-button-shape-next--size-m yt-spec-button-shape-next--icon-leading yt-spec-button-shape-next--segmented-start','aria-label':True}) if like_button: like_count = like_button.get('aria-label') except : like_count = '' try: #用户头像 user_img = '' img_div = soup.find('div',attrs={'id':'owner'}) if img_div: img_img = img_div.find('img',attrs={'id':'img','src':True}) if img_img: user_img = img_img.get('src') except : user_img = '' try: #粉丝 fans_count = '' fans_count_div = soup.find('yt-formatted-string',attrs={'id':'owner-sub-count','aria-label':True}) if fans_count_div: fans_count = fans_count_div.text.strip() except : fans_count = '' try: #用户个人链接 author_link = '' user_link_a = soup.find('a',attrs={'class':'yt-simple-endpoint style-scope ytd-video-owner-renderer','href':True}) if user_link_a: author_link = user_link_a.get('href') author_link = "https://www.youtube.com" + author_link except: author_link = '' try: #描述 describe = '' desc_spans = soup.find_all('span',attrs={'dir':'auto','class':'yt-core-attributed-string--link-inherit-color'}) if len(desc_spans)>0: for desc in desc_spans: describe = describe + '\n' + desc.text.strip().replace('\n','').replace('\t','') except: describe = '' try: #评论数 comment_counts = '' comment_counts_div = soup.find('yt-formatted-string',attrs={'class': 'count-text style-scope ytd-comments-header-renderer'}) if comment_counts_div: comment_text = comment_counts_div.text.strip() print(f'comment_text:{comment_text}') # 使用正则表达式提取数字部分 numbers_str = re.findall(r'\d+', comment_text) # 将提取到的数字字符串连接起来 comment_counts = ''.join(numbers_str) except: comment_counts = '' entertainment_type = 'video' detail_type = 'video' creator_id = utils.str_to_md5(author) creator_img = user_img creator_img_id = f'{creator_id}.jpg' video_img_url = video_img_link video_img_id = f'{id}.jpg' if duration: try: minutes = int(duration.split(':')[0]) if minutes <= 4: print(f'该视频时长为:{duration},小于5分钟，可以采集视频') except ValueError: print(f"警告：无法解析视频时长 '{duration}'") else: print(f"警告：视频缺少时长信息") run_scrolling(page=page,comment_counts=int(comment_counts), min_scroll_distance=500, max_scroll_distance=800, min_wait_time=1, max_wait_time=2) """加载 YouTube 视频的所有评论""" # while True: # # 查找所有“加载更多”按钮 # load_more_buttons = page.locator("button[class*='yt-spec-button-shape-next yt-spec-button-shape-next--text yt-spec-button-shape-next--call-to-action yt-spec-button-shape-next--size-m yt-spec-button-shape-next--icon-leading']") # # 检查按钮是否存在 # if load_more_buttons.count() > 0: # # 遍历所有按钮并点击 # for button in load_more_buttons.all_inner_texts(): # if button.text_content() == '加载更多': # button.click() # # # 等待页面加载更多评论 # page.wait_for_timeout(2000) # # else: # # 没有找到“加载更多”按钮，说明所有评论都已加载 # break # multimedia_info = { # # "creator_id": creator_id, # "creator_img": creator_img, # "creator_img_id": creator_img_id, # "original": author, # "tag": tag, # "describe": describe, # "video_img_url": video_img_url, # "video_img_id": video_img_id, # } print(f'封面链接:{video_img_url}') video_info = VideoInfo(entertainment_type=entertainment_type, detail_type=detail_type, url=url, title=title, view_count=view_count, comment_count=comment_counts, duration=duration, description=describe, pub_time = pub_time, username=author, user_link = author_link, like_count=like_count ) file_ext = '.jpg' video_img_id = utils.str_to_md5(video_img_url) file_temp_list = [] file = File(file_url=video_img_url, file_type='img', file_ext='.jpg', name=f'{video_img_id}{file_ext}', src_col='ent_infos', src_id=utils.str_to_md5(url), src_url=url, title=title ) file_temp_list.append(file) file_entities = {'files': file_temp_list} file_product = CommonModel(spider=self, big_class=BigClass.VIDEO, entities=file_entities) file_product.save_img_file(img_url=video_img_url) # 临时列表初始化 file_temp_list = [] files.append(file) # 写入实体列表 comment_entities = {"video_infos": [video_info]} product = CommonModel(spider=self, big_class=BigClass.VIDEO,entities=comment_entities) product.save_file() print(f'entertainment_info json str:{video_info.__dict__}') break # 成功则退出重试 except Exception as e: retry_count += 1 print(f'解析视频详情页失败，重试次数 {retry_count}/{max_retries}: {traceback.format_exc()}') if retry_count >= max_retries: print(f'达到最大重试次数，放入队列，重新处理该页面: {url}') remake_dict = { "post_url": url, "video_img_link": video_img_link, "duration": duration, "title": title, "tag": tag, "author": author, "pub_time": str(pub_time), "view_count": view_count } dao.put_spider_work_queue(self.name, json.dumps(remake_dict)) def handle_response(self, response: Response, url: str): if "https://www.youtube.com/youtubei/v1/next?prettyPrint=false" in response.url: if response.status == 200: content = response.body() if not content: print(f"来自 {response.url} 的响应为空") return try: decoded_content = content.decode('utf-8') data = json.loads(decoded_content) # 检查 data 是否为字典 if not isinstance(data, dict): print(f"警告：响应数据不是字典类型: {type(data)}") return # 如果不是字典，则跳过处理 # 解析 JSON 数据 mutations = data["frameworkUpdates"]["entityBatchUpdate"]["mutations"] if mutations and len(mutations) > 0: # 确保mutations不为空 video_comments = [] # 用于存储 entertainment_comment_info 对象的列表 video_users = [] # 用于存储 video_user 对象的列表 files_list = [] # 用于存储 file 对象的列表 save_interval = 10 # 定义保存间隔 for i, mutation in enumerate(mutations): # 遍历mutations # 初始化变量（避免未定义错误） youtube_comment_id = '' content = '' published_time = '' level = '' author = '' user_avatar_url = '' author_link = '' like_count = '' reply_count = '' payload = mutation.get("payload") if not payload or not isinstance(payload, dict): continue # 跳过无效payload # 只处理包含 commentEntityPayload 的 mutation（其他类型忽略） comment_entity_payload = payload.get("commentEntityPayload") if not comment_entity_payload or not isinstance(comment_entity_payload, dict): continue # 提取子字段（带空值判断） properties = comment_entity_payload.get("properties", {}) # 默认为空字典 author_data = comment_entity_payload.get("author", {}) # 默认为空字典 toolbar_data = comment_entity_payload.get("toolbar", {}) # 默认为空字典 # 解析具体字段（层层判断，避免空值错误） youtube_comment_id = properties.get('commentId', '') # 解析内容（处理嵌套字典） content_obj = properties.get('content', {}) content = content_obj.get('content', '') published_time = properties.get('publishedTime', '') level = properties.get('replyLevel', '') author = author_data.get('displayName', '') user_avatar_url = author_data.get('avatarThumbnailUrl', '') # 解析作者链接（多层嵌套） channel_command = author_data.get('channelCommand', {}) innertube_cmd = channel_command.get('innertubeCommand', {}) cmd_metadata = innertube_cmd.get('commandMetadata', {}) web_cmd_meta = cmd_metadata.get('webCommandMetadata', {}) author_link = web_cmd_meta.get('url', '') # 解析点赞数和回复数 like_count = toolbar_data.get('likeCountNotliked', '') reply_count = toolbar_data.get('replyCount', '') user_avatar_id = utils.str_to_md5(user_avatar_url) user_name = author user_id = utils.str_to_md5(user_name) user_img_id = f'{user_id}.jpg' pub_time = convert_relative_time(published_time) user_url = self.channel_url + author_link video_comment = VideoComment(username=author, user_url=user_url, pub_time=pub_time, comment=content, like_count=like_count, reply_count=reply_count, post_url=url, post_id=utils.str_to_md5(url), level=level, youtube_comment_id=youtube_comment_id) video_user = SocialUser(user_url=user_url, username=author, nickname=author ) file_ext = '.jpg' user_avatar_id = utils.str_to_md5(user_avatar_url) file_temp_list = [] file = File(file_url=user_avatar_url, file_type='img', file_ext='.jpg', name=f'{user_avatar_id}{file_ext}', src_col='video_users', src_id=utils.str_to_md5(user_url), src_url=user_url ) file_temp_list.append(file) file_entities = {"files": file_temp_list} file_product = CommonModel(spider=self, big_class=BigClass.VIDEO, entities=file_entities) file_product.save_img_file(img_url=user_avatar_url) #临时列表初始化 file_temp_list = [] video_comments.append(video_comment) video_users.append(video_user) files_list.append(file) if len(video_comments) >= save_interval: comment_entities = {"video_comments": video_comments, "video_users": video_users} product = CommonModel(spider=self, big_class=BigClass.VIDEO, entities=comment_entities) print(f'video_comment_infos:{video_comments}') product.save_file() # 初始化列表 video_comments = [] video_users = [] files_list = [] if video_comments: # 如果数据不够一组(批量保存)，保存剩余的数据 comment_entities = {"video_comments": video_comments, "video_users": video_users} product = CommonModel(spider=self, big_class=BigClass.VIDEO, entities=comment_entities) print(f'entertainment_comment_infos:{video_comments}') product.save_file() except json.JSONDecodeError as e: print(f"来自 {response.url} 的响应包含无效的JSON数据: {e}") except UnicodeDecodeError as e: print(f"解码 {response.url} 响应时出错: {e}") except Exception as e: print(f"处理 {response.url} 响应时发生错误: {traceback.format_exc()}") else: print(f"请求 {response.url} 失败，状态码为 {response.status}") print(response.text) # 打印响应文本，方便调试 def handle_search(self, page): """原搜索逻辑不变，爬取首页链接存入Redis""" utils.log_print("开始爬取首页链接...") page.goto("https://www.youtube.com/feed/trending?bp=6gQJRkVleHBsb3Jl", wait_until='domcontentloaded', timeout=60000) time.sleep(random.randint(5, 10)) # 等待页面加载 # 滚动加载更多内容（保持原逻辑） # try: # run_scrolling(page=page, comment_counts=0) # 滚动参数可调整 # except Exception as e: # utils.log_print(f"滚动页面出错：{e}") page.evaluate(f"window.scrollBy(0, 1000);") # 解析页面提取链接（保持原逻辑） try: html = page.content() soup = BeautifulSoup(html, 'lxml') item_links = soup.find_all('div', attrs={'id': 'dismissible', 'class': 'style-scope'}) for item in item_links: video_link_a = item.find('a',attrs={'id':'video-title','href':True}) if video_link_a: video_link = video_link_a['href'] if '/shorts/' not in video_link: item_link = f"https://www.youtube.com{video_link}" #封面 img_link = '' img = item.find('img',attrs={'src':True}) if img: img_link = img['src'] #时长 duration = '' duration_div = item.find('div',class_ = 'badge-shape-wiz__text') if duration_div: duration = duration_div.text.strip() #标题 title = '' title = video_link_a.text.strip() #标签 tag = '' tag_div = item.find('div',attrs={'id':'tooltip','class':'style-scope tp-yt-paper-tooltip hidden','style-target':'tooltip'}) if tag_div: tag = tag_div.text.strip() #作者 author = '' author_div = item.find('div',attrs={'id':'text-container','class':'style-scope ytd-channel-name'}) if author_div: author = author_div.text.strip() item_views = item.find('div',attrs={'id':'metadata-line','class':'style-scope ytd-video-meta-block'}) view_count = '' pub_time = '' if item_views: spans = item_views.find_all('span',recursive = False) if len(spans) == 2: views = spans[1].text.strip() # 发布时间 pub_time = convert_relative_time(views) # 播放量 view_count = spans[0].text.strip() #print(f'item_link:{item_link}') post_dict = { "post_url": item_link, "video_img_link": img_link, "duration": duration, "title": title, "tag": tag, "author": author, "pub_time": str(pub_time), "view_count": view_count } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) utils.log_print(f"已存入任务：{item_link}") except Exception as e: utils.log_print(f"解析首页链接出错：{traceback.format_exc()}") from dateutil.relativedelta import relativedelta from datetime import datetime import re def convert_relative_time(time_str): """ 将相对时间字符串转换为 datetime 对象，支持年月日时分秒的格式。 Args: time_str: 相对时间字符串，例如“1年2月3天4小时5分钟6秒前”。 Returns: datetime 对象，表示 time_str 对应的时间。如果无法解析 time_str，则返回 None。 """ try: total_years = 0 total_months = 0 total_days = 0 total_hours = 0 total_minutes = 0 total_seconds = 0 # 匹配年、月、日、时、分、秒 year_match = re.search(r"(\d+)年", time_str) if year_match: total_years = int(year_match.group(1)) month_match = re.search(r"(\d+)个月", time_str) if month_match: total_months = int(month_match.group(1)) day_match = re.search(r"(\d+)天", time_str) if day_match: total_days = int(day_match.group(1)) hour_match = re.search(r"(\d+)小时", time_str) if hour_match: total_hours = int(hour_match.group(1)) minute_match = re.search(r"(\d+)分钟", time_str) if minute_match: total_minutes = int(minute_match.group(1)) second_match = re.search(r"(\d+)秒", time_str) if second_match: total_seconds = int(second_match.group(1)) # 计算总时间差 delta = relativedelta(years=total_years, months=total_months, days=total_days, hours=total_hours, minutes=total_minutes, seconds=total_seconds) now = datetime.now() datetime_ymdhms = datetime(now.year, now.month, now.day, now.hour, now.minute) return str(datetime_ymdhms - delta) except Exception as e: print(f"解析时间字符串出错: {e}") return None def run_scrolling(page,comment_counts,min_scroll_distance=300,max_scroll_distance = 500,min_wait_time = 1,max_wait_time = 3,stop_probability = 0.1): # max_scroll_times = 100 #最大滚动次数 # max_scroll_distance = 500 #最大滚动距离 # min_wait_time = 500 # 最小等待时间（毫秒） # max_wait_time = 2000 # 最大等待时间（毫秒） # stop_probability = 0.1 # 停止概率 # comment_counts 评论数 previous_scroll_position = 0 min_wait_time = min_wait_time * 1000 max_wait_time = max_wait_time * 1000 scroll_count = 0 scroll_height = 0 # 初始化页面高度 error_count = 0 # 初始化错误计数器 max_errors = 10 # 最大容错次数 tolerance=10 check_interval=0.5 # 计算滚动次数 if comment_counts is None or not isinstance(comment_counts, int) or comment_counts <= 0: print("评论数无效，默认滚动次数100次。") max_scroll_times = 100 else: max_scroll_times = comment_counts // 50 # 整除10 max_scroll_times = max_scroll_times + 150 if comment_counts % 10 != 0: # 如果有余数，则加1 max_scroll_times += 1 print(f"评论总数：{comment_counts}，计算出的滚动次数：{max_scroll_times}") while scroll_count < max_scroll_times: # 随机滚动距离 scroll_distance = random.randint(min_scroll_distance, max_scroll_distance) print(f"目前滚动次数：{scroll_count}，最大滚动次数：{max_scroll_times}") # 模拟加速和减速 if scroll_count < max_scroll_times / 3: scroll_distance = int(scroll_distance * 1.5) # 前三分之一加速 elif scroll_count > max_scroll_times * 2 / 3: scroll_distance = int(scroll_distance * 0.5) # 后三分之一减速 page.evaluate(f"window.scrollBy(0, {scroll_distance});") # 随机等待时间 time.sleep(random.randint(min_wait_time, max_wait_time) / 1000) #wait_time = random.randint(min_wait_time, max_wait_time) / 1000 #page.wait_for_timeout(wait_time) # 间歇性停止 if random.random() < stop_probability: stop_time = random.randint(1000, 3000) / 1000 # 停止1到3秒 page.wait_for_timeout(stop_time) # 等待页面加载完成 (可以根据实际情况调整等待时间) page.wait_for_load_state("networkidle") # 等待网络空闲 # new_scroll_height = page.evaluate("document.body.scrollHeight") # # 增加判断的严谨性 # if abs(new_scroll_height - scroll_height) <= tolerance: # page.wait_for_timeout(check_interval * 1000) # check_new_scroll_height = page.evaluate("document.body.scrollHeight") # if abs(check_new_scroll_height - scroll_height) <= tolerance: # error_count += 1 # if error_count >= max_errors: # print(f"页面高度连续{max_errors}次未变化，跳出循环") # break # else: # error_count = 0 # else: # error_count = 0 # scroll_height = new_scroll_height # 更新页面高度 scroll_count += 1