编辑采集器       终止采集器

属性	值	属性	值
名称	us_facebook_1	创建时间	2025-04-02 10:10:02
标题	美国-Facebook	更新时间	-
状态	running	开始时间	2025-07-04 00:26:07
停止类型		结束时间	-
信号		运行时长	89.5小时
进程ID	3829	进程名称	Process-3
作业ID	1753b82e-48c9-4157-a1cb-2f8389c84363	作业目录	records/jobs/1753b82e-48c9-4157-a1cb-2f8389c84363
日志文件	records/logs/1753b82e-48c9-4157-a1cb-2f8389c84363.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_facebook_1_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import json import os import platform import random import time import traceback from enum import Enum from playwright.sync_api import sync_playwright, expect import dao import utils from spider.spiders.common.base_spider import BaseListToDetailsSpider from spider.spiders.common.models import Language, Nation # 创建临时目录 user_home = os.path.expanduser('~') temp_dir = os.path.join(user_home, 'temp_data_video') if not os.path.exists(temp_dir): os.makedirs(temp_dir) class CrawlerType(Enum): SEARCH = 'keywords' # 按关键词搜索 USER = 'accounts' # 以用户ID为入口 POST = 'posts' # 以推文url为入口 TAG = 'tag' # 按话题搜索 class UsFacebook1Spider(BaseListToDetailsSpider): name = 'us_facebook_1' desc = '美国-Facebook' net_name = 'Facebook' net_url = 'www.facebook.com' language = Language.EN nation_category = Nation.OUT_SIDE channel_name = 'FacebookSearch' channel_url = 'https://www.facebook.com/search/posts' channel_desc = 'Facebook>FacebookSearch' # 已处理推文 processed_post = {} # 已处理username processed_username = {} # 已处理视频链接 processed_url = {} # 推文是否有视频 post_has_video = {} user_dict = {} # 总处理推文数 sum = 0 # 重复推文数 repeat_num = 0 # 成功处理推文数 success_num = 0 # 跳过数量 skip_num = 0 # 广告数 ad_num = 0 # 错误数（包含广告和其他错误） error_num = 0 # 图像数量 image_num = 0 # 头像数量 profile_num = 0 # 视频数量 video_num = 0 # ZIP数量 zip_num = 0 # 其他文件数量 txt_num = 0 cmd_id = 0 # 最大翻页数 max_page = 100 # 评论最大深度 max_depth = 3 timeout_count = 0 # 读取队列最大超时次数 max_timeout_count = 6 # 是否按用户ID发散 diverge_user = False cookies = [ { "domain": ".facebook.com", "expirationDate": 1775184016.923824, "hostOnly": False, "httpOnly": True, "name": "datr", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "lNC_Z9ALyS0-TUdPXVbiSb9S" }, { "domain": ".facebook.com", "expirationDate": 1775186938.106767, "hostOnly": False, "httpOnly": True, "name": "sb", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "lNC_Z6AEuiBE5ZytPkdvwh11" }, { "domain": ".facebook.com", "expirationDate": 1775184674.684634, "hostOnly": False, "httpOnly": True, "name": "ps_l", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "1" }, { "domain": ".facebook.com", "expirationDate": 1775184674.684785, "hostOnly": False, "httpOnly": True, "name": "ps_n", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "1" }, { "domain": ".facebook.com", "expirationDate": 1741231414.437295, "hostOnly": False, "httpOnly": False, "name": "locale", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "en_US" }, { "domain": ".facebook.com", "expirationDate": 1772162938.106385, "hostOnly": False, "httpOnly": False, "name": "c_user", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "61566383793097" }, { "domain": ".facebook.com", "expirationDate": 1772162938.106901, "hostOnly": False, "httpOnly": True, "name": "xs", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "41%3A9JKUuHcpWrDPlg%3A2%3A1740626940%3A-1%3A-1" }, { "domain": ".facebook.com", "expirationDate": 1748402938.458424, "hostOnly": False, "httpOnly": True, "name": "fr", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "1MSWEAnGiglrcm0Jq.AWXBnxp6QtmGIyrGQIP_sj4MFrkzF5I4vG-0ZQ.Bnv9Mz..AAA.0.0.Bnv9v-.AWVnf5gSw4s" }, { "domain": ".facebook.com", "expirationDate": 1741231743, "hostOnly": False, "httpOnly": False, "name": "wd", "path": "/", "secure": True, "session": False, "storeId": "0", "value": "571x617" }, { "domain": ".facebook.com", "hostOnly": False, "httpOnly": False, "name": "presence", "path": "/", "secure": True, "session": True, "storeId": "0", "value": "C%7B%22t3%22%3A%5B%5D%2C%22utc3%22%3A1740626945266%2C%22v%22%3A1%7D" } ] # 关键词搜索 @staticmethod def get_search_url(keyword): return f'https://www.facebook.com/search/posts?q={keyword}' # 用户主页 @staticmethod def get_user_url(user_id): return f'https://www.facebook.com/{user_id}' # 话题搜索 @staticmethod def get_search_tag_url(tag): return f'https://www.facebook.com/hashtag/{tag}' # 默认搜索关键词 @staticmethod def get_default_targets(): # return ['Philippines', 'Scarborough Shoal', 'Second Thomas Shoal', 'Thitu Island', 'Fiery Cross Reef', 'Subi Reef', 'Mischief Reef', 'Sabina Shoal'] keywords = ['Philippines', 'Scarborough Shoal', 'Second Thomas Shoal', 'Thitu Island', 'Fiery Cross Reef', 'Subi Reef', 'Mischief Reef', 'Sabina Shoal'] keyword = random.choice(keywords) return [keyword] # 开始爬取 def start_requests(self): # 创建临时目录 user_home = os.path.expanduser('~') temp_dir = os.path.join(user_home, 'temp_data_video') if not os.path.exists(temp_dir): os.makedirs(temp_dir) # 读取待处理命令 cmd_info_op = CrawlerType.SEARCH.value cmd_infos = dao.get_spider_cmd_wait_infos(self.name) if not cmd_infos: utils.log_print(f"没有待处理的命令") targets = self.get_default_targets() else: # 拿取创建时间最早的命令 cmd_info = cmd_infos[0] utils.log_print(f'开始爬取, cmd_info: {cmd_info}') self.cmd_id = cmd_info.get('id') dao.update_spider_cmd_info(self.name, self.cmd_id, 'create_time', time.time()) cmd_info_op = cmd_info.get('op') cmd_info_str = cmd_info.get('cmd_str') targets = cmd_info_str.split('\r\n') # 打开浏览器爬取 with sync_playwright() as p: headless = False if platform.system() == 'Windows' else True browser = p.chromium.launch(headless=headless) context = browser.new_context() context.add_cookies(self.cookies) if CrawlerType.SEARCH.value == cmd_info_op: for target in targets: self.handle_search(context, target) if CrawlerType.TAG.value == cmd_info_op: for target in targets: self.handle_tag(context, target) elif CrawlerType.USER.value == cmd_info_op: for target in targets: self.handle_user(context, target) elif CrawlerType.POST.value == cmd_info_op: for target in targets: self.handle_post(target) else: utils.log_print(f"op参数不支持：{cmd_info_op}") utils.log_print( f'sum: {self.sum}, repeat_num: {self.repeat_num}, success_num: {self.success_num}, ad_num: {self.ad_num}, skip_num: {self.skip_num}, error_num: {self.error_num}') utils.log_print( f'zip_num: {self.zip_num}, image_num: {self.image_num}, profile_num: {self.profile_num}, video_num: {self.video_num}, txt_num: {self.txt_num}') self.update_count() dao.update_spider_cmd_info(self.name, self.cmd_id, 'stop_time', time.time()) dao.update_spider_cmd_info(self.name, self.cmd_id, 'status', 'stop') return [] def update_count(self): dao.update_spider_cmd_info(self.name, self.cmd_id, 'page_count', self.sum) dao.update_spider_cmd_info(self.name, self.cmd_id, 'txt_count', self.txt_num) dao.update_spider_cmd_info(self.name, self.cmd_id, 'zip_count', self.zip_num) dao.update_spider_cmd_info(self.name, self.cmd_id, 'img_count', self.image_num) dao.update_spider_cmd_info(self.name, self.cmd_id, 'prof_img_count', self.profile_num) dao.update_spider_cmd_info(self.name, self.cmd_id, 'video_count', self.video_num) data = f'page_count:{self.sum}, txt_count:{self.txt_num}, zip_count:{self.zip_num}, img_count:{self.image_num}, prof_img_count:{self.profile_num}, video_count:{self.video_num}' # todo utils.log_print(f'同步更新zk, data: {data}') # 处理关键字搜索 def handle_search(self, context, target): self.parse_posts(context, self.get_search_url(target), target) # 处理话题搜索 def handle_tag(self, context, target): self.parse_posts(context, self.get_search_tag_url(target), target) # 处理用户ID搜索 def handle_user(self, context, user_id): self.parse_posts(context, self.get_user_url(user_id), user_id) # 处理帖子链接搜索 def handle_post(self, target): self.sum += 1 if self.sum % 10 == 0: self.update_count() if self.processed_post.get(target): self.repeat_num += 1 utils.log_print(f'推文已重复，processed_post: {target}') else: utils.log_print(f'推文未重复，publish_url: {target}') self.processed_post[target] = time.time() post_dict = { "post_url": target, "keyword": target, "parent_url": None, "root_url": target, "depth": 1, } utils.log_print(f"推文加入队列: {post_dict}") dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 解析帖子区域，找posts的链接使用模糊匹配的方式，因为帖子链接元素的class一直会发送变化，所以不能直接用xpath def parse_posts(self, context, url, target): urls_count = 0 # 帖子区域选择符 item_selector_str = 'div[class="x1yztbdb x1n2onr6 xh8yej3 x1ja2u2z"]' # 打开页面 page = context.new_page() print(f'打开页面：{url}') start_time = time.time() page.goto(url, timeout=300000, wait_until='load') print(f'页面加载完成：{url}，耗时：{time.time() - start_time}s') # 解析帖子列表，最大解析个数=每页帖子数*最大翻页数 items = page.locator(item_selector_str) for article_num in range(items.count() * self.max_page): try: # 帖子区域可见（如果没有这么多帖子，则说明已经到达最后一页），抛出超时异常 expect(page.locator(item_selector_str).nth(article_num)).to_be_visible(timeout=60000) # 解析帖子链接，需要hover才会显示完整链接 post_urls = items.nth(article_num).locator('a') if post_urls.count() == 0: continue for post_url_num in range(post_urls.count()): time.sleep(random.uniform(1, 3)) post_url = post_urls.nth(post_url_num) if not post_url.is_visible() or not post_url.is_enabled(): continue post_url.scroll_into_view_if_needed() post_url.hover(force=True) url = post_url.get_attribute('href') # 暂时只找帖子链接，忽略group，reels，live等链接 if url and '/posts' in url: print(f'发现新帖子: {url}') self.parse_post_details(context, url) urls_count += 1 except: print( f'解析帖子列表异常（可能已经到达最后一页），article_num: {article_num}，exception: {traceback.format_exc()}') print(f'共发现{urls_count}个帖子') page.close() # 解析帖子详情 def parse_post_details(self, context, post_url): print(f'开始解析帖子详情: {post_url}') page = context.new_page() start_time = time.time() page.goto(post_url, timeout=60000, wait_until='load') print(f'页面加载完成：{post_url}，耗时：{time.time() - start_time}s') # 内容区域的 XPath 表达式 context_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[3]' image_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[3]' like_count_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[4]/div/div/div[1]/div/div[1]/div/div[1]/div/span/div/span[1]' comment_count_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[4]/div/div/div[1]/div/div[1]/div/div[2]/div[2]/span/div/span/span' share_count_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[4]/div/div/div[1]/div/div[1]/div/div[2]/div[3]/span/div/span/span' comment_area_xpath = '/html/body/div[1]/div/div[1]/div/div[5]/div/div/div[2]/div/div/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div/div/div/div/div/div/div/div/div[13]/div/div/div[4]/div/div/div[2]/div[3]' try: # 内容 context_element = page.locator(f'xpath={context_xpath}') text_content = '' if context_element.count() != 0: # 获取元素的文本内容 text_content = context_element.nth(0).text_content() # 图片 image_urls = [] image_element = page.locator(f'xpath={image_xpath}') img_tags = image_element.locator('img') img_count = img_tags.count() if img_count > 0: for i in range(img_count): image_urls.append(img_tags.nth(i).get_attribute('src')) # 点赞数 like_element = page.locator(f'xpath={like_count_xpath}') like_count = 0 if like_element.count() != 0: like_count = like_element.text_content().replace(',', '') # 评论数 comment_element = page.locator(f'xpath={comment_count_xpath}') comment_count = 0 if comment_element.count() != 0: comment_count = comment_element.text_content().replace(',', '') # 分享数 share_element = page.locator(f'xpath={share_count_xpath}') share_count = 0 if share_element.count() != 0: share_count = share_element.text_content().replace(',', '') # 评论区 comments = [] try: comment_area_element = page.locator(f'xpath={comment_area_xpath}') comment_children_element = comment_area_element.locator("> *") comment_item_count = comment_children_element.count() for i in range(comment_item_count): comment_element = comment_children_element.nth(i) # 假设子元素内要查找的元素的相对 XPath user_url_xpath = './/div/div[1]/div[2]' user_avatar_xpath = './/div/div[1]/div/div[1]' nickname_xpath = './/div/div[1]/div[2]' comment_content_xpath = './/div/div[1]/div/div[2]' # 在子元素内根据相对 XPath 查找元素 user_url_element = comment_element.locator(f'xpath={user_url_xpath}') user_url = user_url_element.locator('a').nth(0).get_attribute('href') user_avatar_element = comment_element.locator(f'xpath={user_avatar_xpath}') user_avatar_url = user_avatar_element.locator('image').get_attribute('xlink:href') nickname_element = comment_element.locator(f'xpath={nickname_xpath}') nickname = '' if nickname_element.count() != 0: nickname = nickname_element.nth(0).text_content() comment_content_element = comment_element.locator(f'xpath={comment_content_xpath}') comment_content = '' if comment_content_element.count() != 0: comment_content = comment_content_element.nth(0).text_content() comment = f'用户链接：{user_url}，昵称：{nickname}，评论内容：{comment_content}，用户头像：{user_avatar_url}' comments.append(comment) print(f"评论：{comment}") except: print(f"解析评论区异常：{traceback.format_exc()}") print("标题：", page.title()) print("内容：", text_content) print(f"图片个数：{image_urls.__len__()}，图片：{image_urls}") print(f"点赞数：{like_count}") print(f"评论数：{comment_count}") print(f"分享数：{share_count}") print(f"评论内容数：{comments.__len__()}") except: print(f"定位元素或获取内容时出错：{traceback.format_exc()}") # 关闭浏览器 page.close()