编辑采集器       终止采集器

属性	值	属性	值
名称	us_tiktok_v2	创建时间	2025-04-02 10:10:02
标题	美国-Tiktok-V2	更新时间	-
状态	running	开始时间	2025-07-04 00:26:41
停止类型		结束时间	-
信号		运行时长	96.3小时
进程ID	3880	进程名称	Process-20
作业ID	e7e509f7-48c7-46dd-869f-38b85a129635	作业目录	records/jobs/e7e509f7-48c7-46dd-869f-38b85a129635
日志文件	records/logs/e7e509f7-48c7-46dd-869f-38b85a129635.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_tiktok_v2_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
f806dba2-fd6a-46d9-b510-aeef134dc680	2025-06-20 10:06:24	2025-06-27 03:57:41	161.9小时	1565941	Process-1	查看日志	删除记录
24e18e09-7974-4f31-98e1-5ca92d243068	2025-06-11 16:58:10	2025-06-13 10:41:52	41.7小时	3083647	Process-1	查看日志	删除记录
624929b1-aa68-4738-8fe0-f0e162ebdbcb	2025-05-19 16:53:39	-	1191.9小时	3858752	Process-22	查看日志	删除记录
d672494b-552a-475c-bd21-d87280b9bd6d	2025-05-16 21:08:14	2025-05-19 16:53:35	67.8小时	2563995	Process-13	查看日志	删除记录
3120d095-d0c1-459e-8430-130d719b7ec1	2025-05-12 17:56:14	2025-05-14 14:42:21	44.8小时	570316	Process-22	查看日志	删除记录
c10a8755-e09b-47e1-9f33-6666a6f348b5	2025-05-08 09:19:29	2025-05-12 15:57:15	102.6小时	3217119	Process-31	查看日志	删除记录
c639e37b-4975-491f-9191-39f8ea7ed5fc	2025-05-06 18:59:38	2025-05-07 17:25:03	22.4小时	2519744	Process-31	查看日志	删除记录
删除最后 选择条数 条历史作业 确定

代码

import time import sys from urllib.parse import unquote from DrissionPage import Chromium from DrissionPage._pages.chromium_tab import ChromiumTab from spider.spiders.common.models import Language, Nation from spider.spiders.common.base_spider import BaseListToDetailsSpider import spider.spiders.resource.tiktok.tiktok_helper as helper import spider.spiders.us_tiktok_v2_cookies as us_tiktok_cookies import utils import dao # us_tiktok_v2_spider class UsTikTokV2Spider(BaseListToDetailsSpider): name = 'us_tiktok_v2' desc = '美国-Tiktok-V2' net_name = 'Tiktok' net_name_zh = '抖音国外版' net_url = 'www.tiktok.com' net_nation = 'US' net_language = 'en' language = Language.EN nation_category = Nation.OUT_SIDE channel_name = 'Top' channel_url = 'https://www.tiktok.com' channel_desc = 'Tiktok>Top' # TODO: remove search_keywords = ['philippine sea','Panatag Shoal','仙宾礁','South China Sea', 'Woody Island','Mischief Reef','Subi Reef','Spratly Island', 'Scarborough Shoal','Thitu Island','Northeast Cay','Fiery Cross Reef', 'Cuarteron Reef','Johnson South Reef','Hughes Reef','','Gaven Reef' 'Itu Aba','Tree Island','North Reef','Beilun Reef', 'Zhaoshu Island','Taiping Island','Nanxun Reef','Dongmen Reef', 'Chigua Reef','Huayang Reef','Yongshu Reef','Nanyao Island', 'Zhongye Island','Huangyan Island','Nanwei Island','Zhubi Reef', 'Meiji Reef','Yongxing Island','永兴岛','美济礁', '渚碧礁','南威岛','黄岩岛','中业岛', '南钥岛','永暑礁','华阳礁','赤瓜礁', '东门礁','南薰礁','太平岛','赵述岛','北仑礁'] count = 0 tab: ChromiumTab = None browser: Chromium = None # 浏览器设定 browser_user_data_path = '' browser_local_port = 51888 browser_headless = False browser_size = '1920,1080' home_url = 'https://www.tiktok.com' runtime_tmp_dir = 'tmp' search_listen_timeout = 20 # 应该 15 - 20 user_listen_timeout = 20 user_follow_listen_timeout = 15 open_wait_seconds = 30 video_count = 0 author_img_count = 0 max_level = 2 max_page = 10000 max_item = 0 max_comment_page = 10 max_comment_item = 0 max_user_post_page = 10 max_user_post_item = 0 max_user_follow_page = 20 max_user_follow_item = 0 post_urls_dict = {} cmd_task_id = 'tiktok_cmd_task_test_01' cmd_task_root_video_index = 0 cmd_task_user_index = 0 # dev: is_dev = False if 'linux' in sys.platform else True if is_dev: max_level = 2 max_page = 1 max_item = 1 max_comment_page = 1 max_comment_item = 1 max_user_post_page = 1 max_user_post_item = 1 max_user_follow_page = 1 max_user_follow_item = 1 search_listen_timeout = 15 user_listen_timeout = 15 open_wait_seconds = 15 def start_requests(self): utils.log_print(f'开始 {self.net_name} 采集, 测试环境: {self.is_dev}') utils.log_print(f'准备初始化浏览器') helper.init_browser(self) tab: ChromiumTab = self.tab browser = self.browser if self.is_dev: browser.set.cookies(us_tiktok_cookies.user_cookies_xafly002) else: browser.set.cookies(us_tiktok_cookies.user_cookies_51888) utils.log_print(f'创建tab和设置cookies完成') time.sleep(2) # 打开首页 tab.get('https://www.tiktok.com') time.sleep(self.open_wait_seconds) # 使用登录 utils.log_print(f'已打开首页') tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_home_010.jpg') try: login_res = helper.login(self) if login_res is False: utils.log_print(f'[登录] ---错误') tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_login_err_010.jpg') return [] except Exception as e: utils.log_print(f'[登录] ---错误: {e}') tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_login_err_020.jpg') return [] if self.is_dev: # 测试 for keyword in self.search_keywords: self.search(keyword) # user_url = 'https://www.tiktok.com/@ziyoufujian' # user_url = 'https://www.tiktok.com/@tanyuxi61' # self.process_user_task(now_level=1, src_url='root', user_url=user_url) utils.log_print(f'关闭浏览器') tab.close() browser.quit() else: try: for keyword in self.search_keywords: self.search(keyword) except Exception as err: utils.log_error(f'主搜索出错: {err}') tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_901_search_error.jpg') finally: time.sleep(10) utils.log_print(f'关闭浏览器') tab.close() browser.quit() return [] def search(self, search_keyword): # 准备搜索 utils.log_print(f'--------------------------------------------------------') utils.log_print(f'准备搜索: {search_keyword}') utils.log_print(f'搜索最大翻页限制: {self.max_page}, 每页采集项目数: {self.max_item}') utils.log_print(f'评论最大翻页限制: {self.max_comment_page}') utils.log_print(f'--------------------------------------------------------') tab: ChromiumTab = self.tab tab.listen.start(f'https://www.tiktok.com/api/search/general/full') # 搜索 timestamp = int(time.time() * 1000) search_url = f'https://www.tiktok.com/search?q={unquote(search_keyword)}&t={timestamp}' utils.log_print(f'开始搜索: {search_url}') tab.get(search_url) tab.wait.eles_loaded('#search-tabs', 120) # tab.wait.doc_loaded() time.sleep(helper.get_random_seconds()) tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_search_020.jpg') # 向下滚动获取视频数据 for page_num in range(self.max_page): data_packets = tab.listen.wait(count=5, timeout=self.search_listen_timeout, fit_count=False) if not data_packets: utils.log_warn(f'返回数据包空,可能是到最后一页了: page_num = {page_num}') tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_search_090_lastpage.jpg') break tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_search_030.jpg') data_packets_len = len(data_packets) utils.log_print(f'数据包数量: {data_packets_len}') item_num = 0 # 获得全部 all_item_list all_item_list = [] for _, data_packet in enumerate(data_packets): item_list = utils.get_dict_value(data_packet.response.body, 'data', []) all_item_list.extend(item_list) item_list_len = len(all_item_list) utils.log_print(f'视频item数量: {item_list_len}, 页数:{page_num + 1}/{self.max_page}') for index, item in enumerate(all_item_list): if not helper.check_post_base_info(self, item): continue item_num += 1 time.sleep(0.3) # 暂停速度 helper.collect_post(self, now_level=1, src_url='', item=item, from_root=True, do_collect_video=True, do_collect_author=1, do_collect_comments=False) utils.log_print( f'== 视频搜索中: ({index}/{item_list_len}) , 页数:{page_num + 1}/{self.max_page}, 本页中视频: {item_num}, 视频数量 {self.video_count} ==') # 每页采集项数限制 (0为全部) if self.max_item and self.max_item > 0: if item_num >= self.max_item: break # 向下滚动下一页 if page_num + 1 < self.max_page: time.sleep(helper.get_random_seconds()) utils.log_print(f'开始滚动') tab.scroll.to_bottom() time.sleep(helper.get_random_seconds(10, 1)) # 搜索完成 utils.log_print(f'OK, 主搜索完成! 搜索POST数量: {len(self.post_urls_dict)}') # 执行子任务 self.seek_and_run_cmd_task() def seek_and_run_cmd_task(self, src_url: str = ''): utils.log_print(f'\n\n === 开始寻找子任务, max_level: {self.max_level} ===\n\n') max_level = self.max_level now_level = 1 for src_level in range(1, max_level): now_level = src_level + 1 src_level_info_key = helper.get_level_key(self, src_level) utils.log_print(f'src_level_info_key: {src_level_info_key}') level_all_num = dao.red.scard(src_level_info_key) utils.log_print(f'开始第 {now_level} 层, 共 {level_all_num} 个 src_url') level_now_num = 0 seek_all_num = 0 seeked = True while seeked: time.sleep(0.1) cursor = 0 cursor, new_scans = dao.red.sscan(src_level_info_key, cursor=cursor, count=5) if not new_scans: utils.log_print(f'第 {now_level} 层任务完成! \n\n\n') seeked = False break seek_src_urls_b = [] seek_src_urls_b.extend(new_scans) seek_src_urls_len = len(seek_src_urls_b) seek_all_num = seek_all_num + seek_src_urls_len for _, src_url_b in enumerate(seek_src_urls_b): src_url = src_url_b.decode() level_now_num += 1 utils.log_print( f'【处理任务】第 {now_level} 层 ({level_now_num}/{level_all_num}) src_url: {src_url}') self.run_src_url_items(now_level, src_url) helper.clear_src_url(self, src_level, src_url) def run_src_url_items(self, now_level: int = 1, src_url: str = ''): src_post_key, src_user_key = helper.get_url_subs_key(self, src_url) sub_post_info_b = dao.red.hgetall(src_post_key) # utils.log_print(f' [子POST任务提取] now_level: {now_level}, 数量: {len(sub_post_info_b)}') for sub_url_b, _ in sub_post_info_b.items(): sub_url = sub_url_b.decode() sub_url_full_info = 1 # TODO: 放入 value中 self.process_post_task(now_level=now_level, src_url=src_url, post_url=sub_url, do_collect_video=sub_url_full_info, do_collect_author=sub_url_full_info) sub_user_info_b = dao.red.hgetall(src_user_key) # utils.log_print(f' [子USER任务提取] now_level: {src_level}, 数量: {len(sub_post_info_b)}') for sub_url_b, _ in sub_user_info_b.items(): sub_url = sub_url_b.decode() sub_url_full_info = 1 # TODO: 放入 value中 self.process_user_task(now_level=now_level, src_url=src_url, user_url=sub_url, from_root=False, do_collect_user_info=sub_url_full_info) time.sleep(10) def process_post_task(self, now_level: int, src_url: str = '', post_url: str = '', from_root=False, do_collect_video=True, do_collect_author: int = 1, do_collect_comments=True): time.sleep(helper.get_random_seconds()) utils.log_print(f'') utils.log_print(f' * 执行POST任务 {post_url}') listen_comment = f'https://www.tiktok.com/api/comment/list' # listen_reply = f'https://www.tiktok.com/api/comment/list/reply' tab: ChromiumTab = self.tab retry = 3 for try_num in range(retry): try: tab.listen.start(listen_comment) time.sleep(helper.get_random_seconds(2, 1)) tab.get(post_url) tab_html = tab.html tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_post_010.jpg') video_raw_json = helper.get_post_raw_json(post_url, tab_html) helper.collect_post(self, now_level, src_url, {'item': video_raw_json}, from_root=from_root, do_collect_video=do_collect_video, do_collect_author=do_collect_author, do_collect_comments=do_collect_comments) return except Exception as err: tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_post_error_091.jpg') utils.log_print(f' [处理POST任务错误] {err}, 重试次数: {try_num + 1}/{retry}') time.sleep(helper.get_random_seconds()) finally: tab.listen.stop() def process_user_task(self, now_level: int, src_url: str = '', user_url: str = '', from_root=False, do_collect_video=True, do_collect_user_info: int = 1): time.sleep(helper.get_random_seconds()) utils.log_print(f'') utils.log_print(f' * 执行USER任务 {user_url}') post_list_listen_url = 'https://www.tiktok.com/api/post/item_list' # 监听用户发文 listen_url = {post_list_listen_url} tab: ChromiumTab = self.tab retry = 3 for try_num in range(retry): try: tab.listen.start(listen_url, method='GET') tab.get(user_url) tab.wait.doc_loaded() time.sleep(helper.get_random_seconds()) tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_user_010.jpg') helper.collect_user(self, now_level, src_url, user_url, from_root=False, do_collect_video=True, do_collect_user_info=1) return except Exception as err: tab.get_screenshot(path=self.runtime_tmp_dir, name='tiktok_user_error_091.jpg') utils.log_print(f' [处理USER任务错误] {err}, 重试次数: {try_num + 1}/{retry}') time.sleep(helper.get_random_seconds()) finally: tab.listen.stop()