编辑采集器       启动采集器

属性	值	属性	值
名称	us_amazon_sg_shop_clone_1	创建时间	2025-05-26 10:25:13
标题	美国-亚马逊-克隆1	更新时间	-
状态	stop	开始时间	2025-06-27 12:07:55
停止类型	finished	结束时间	2025-06-27 12:17:30
信号		运行时长	9.6分钟
进程ID	4043486	进程名称	Process-1
作业ID	a540d60d-63fa-44b6-8f99-924883d3f537	作业目录	records/jobs/a540d60d-63fa-44b6-8f99-924883d3f537
日志文件	records/logs/a540d60d-63fa-44b6-8f99-924883d3f537.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_amazon_sg_shop_clone_1_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import os import platform import random import threading import time import traceback from bs4 import BeautifulSoup from playwright.sync_api import sync_playwright import dao import utils from spider.spiders.common.base_spider import BaseListToDetailsSpider, logger from spider.spiders.common.models_v2 import File, BuyProduct, CommonModel, BigClass # 创建临时目录 user_home = os.path.expanduser('~') temp_dir = os.path.join(user_home, 'temp_data_video') if not os.path.exists(temp_dir): os.makedirs(temp_dir) utils.log_print(f'--- 临时目录: {temp_dir}') import json class UsAmazonSGShopClone1Spider(BaseListToDetailsSpider): name = "us_amazon_sg_shop_clone_1" desc = "美国-亚马逊-克隆1" net_name = 'amazon' net_url = 'www.amazon.sg' net_name_zh = '亚马逊' net_nation = '美国' nation_category = 1 net_language = 'en' channel_name = 'daily recommendation' channel_url = 'https://www.amazon.com' channel_desc = 'amazon' country_pattern = 'US' platform_name = 'Amazon' platform_name_zh = '亚马逊' platform_url = 'https://www.amazon.com' platform_belong_country = '美国' all_final_urls = [] cookies_list = [ [{'domain': '.amazon.com', 'expirationDate': 1779777863.931318, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.272107, 'hostOnly': False, 'httpOnly': True, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'hwaimcm+dAL0DB7UzMd6oTQM9a/FmEFDGNHtTICAFpyqAV5bgopledSY6pr7GPdNEwTW6gFxpgP9JUgBEFl7uqDCh6CBb1pSBsfwvNn+LDLL3IJCU5F2X9sM2vjbb72gb0/ijjnSPYkNLORZ8owAh6J9hiiUVDx6cWEBzeMOHg/IHCMEzph5VuufRw9757PH1Zvy8McKaaY1nGWm6ZKQz3GnrLBm3ce76nsh78GTpGRnPN/CM8cVtuqfB+HLlhQmNFpBhXSRXQChIkTism75mLPSwzxyCeP5tkclWgHUFBiTAIJpyXL5V8iq5O3l2DIa2Fm05JCcEudvAQqO0vvXSg2JjROylmxw'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778482620, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:K72JTRMB3K65RVB63W6D+s-8A1829Q90YS6J7YM019P|1748242620537&t:1748242620537&adb:adblk_yes'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.27208, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.271978, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.27205, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}], [{'domain': '.amazon.com', 'expirationDate': 1779777863.931318, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1782802809.415889, 'hostOnly': False, 'httpOnly': True, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'MIUKWCvegsiU9t8cE/1Q+JFFZgz53DIbyHXmjuA4qeAkY0sQkiR/1yLEfWCHOUOlodGfD+atFj+tse2EbhgtCwA8iRQdppP13HtmvbEG4iJiFGRUlFL7IjwLlmajrbWzsVhKDrAaZMcqy00IzJ8luGjTqHYDhoACDeNsVvGHVyCXIMxDFBEZ7dhGnXZhVgVIbOTSR2aBXIONmQNY2imOP5Lb2otUbS0z1oXsUtzsgEXEnqDMBwn5Uu8BRYw6VpObXVQhV5e/qMTPKPPe8/axCs0WRrs1yA8cFVAC9eSpdMbble+Cz6h5C+NLVHVm7tLGyNpjeBpgGTmsfxupvv4wtkcpFGp5kRHj'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778482813, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:s-MRHVQA71HN3QW3QPKX07|1748242813075&t:1748242813390&adb:adblk_yes'}, {'domain': '.amazon.com', 'expirationDate': 1782802809.41583, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.271978, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1782802809.415641, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}], [{'domain': '.amazon.com', 'expirationDate': 1779778920.293752, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1782802809.415889, 'hostOnly': False, 'httpOnly': True, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'MIUKWCvegsiU9t8cE/1Q+JFFZgz53DIbyHXmjuA4qeAkY0sQkiR/1yLEfWCHOUOlodGfD+atFj+tse2EbhgtCwA8iRQdppP13HtmvbEG4iJiFGRUlFL7IjwLlmajrbWzsVhKDrAaZMcqy00IzJ8luGjTqHYDhoACDeNsVvGHVyCXIMxDFBEZ7dhGnXZhVgVIbOTSR2aBXIONmQNY2imOP5Lb2otUbS0z1oXsUtzsgEXEnqDMBwn5Uu8BRYw6VpObXVQhV5e/qMTPKPPe8/axCs0WRrs1yA8cFVAC9eSpdMbble+Cz6h5C+NLVHVm7tLGyNpjeBpgGTmsfxupvv4wtkcpFGp5kRHj'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778482926, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:K72JTRMB3K65RVB63W6D+s-C1X6KN0PWP3ZSNEM4HJN|1748242926313&t:1748242926313&adb:adblk_no'}, {'domain': '.amazon.com', 'expirationDate': 1779778920.293843, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802330.271978, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1779778920.293872, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}], [{'domain': '.amazon.com', 'expirationDate': 1779778984.80141, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1782802977.838075, 'hostOnly': False, 'httpOnly': True, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'MIUKWCvegsiU9t8cE/1Q+JFFZgz53DIbyHXmjuA4qeAkY0sQkiR/1yLEfWCHOUOlodGfD+atFj+tse2EbhgtCwA8iRQdppP13HtmvbEG4iJiFGRUlFL7IjwLlmajrbWzsVhKDrAaZMcqy00IzJ8luGjTqHYDhoACDeNsVvGHVyCXIMxDFBEZ7dhGnXZhVgVIbOTSR2aBXIONmQNY2imOP5Lb2otUbS0z1oXsUtzsgEXEnqDMBwn5Uu8BRYw6VpObXVQhV5e/qMTPKPPe8/axCs0WRrs1yA8cFVAC9eSpdMbble+Cz6h5C+NLVHVm7tLGyNpjeBpgGTmsfxupvv4wtkcpFGp5kRHj'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778482982, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:K72JTRMB3K65RVB63W6D+s-HZTPX6P3GZEJ60607AX3|1748242982995&t:1748242982995&adb:adblk_no'}, {'domain': '.amazon.com', 'expirationDate': 1779778984.801509, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802977.837895, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1779778984.801547, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}], [{'domain': '.amazon.com', 'expirationDate': 1779779051.563201, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1782802977.838075, 'hostOnly': False, 'httpOnly': True, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'MIUKWCvegsiU9t8cE/1Q+JFFZgz53DIbyHXmjuA4qeAkY0sQkiR/1yLEfWCHOUOlodGfD+atFj+tse2EbhgtCwA8iRQdppP13HtmvbEG4iJiFGRUlFL7IjwLlmajrbWzsVhKDrAaZMcqy00IzJ8luGjTqHYDhoACDeNsVvGHVyCXIMxDFBEZ7dhGnXZhVgVIbOTSR2aBXIONmQNY2imOP5Lb2otUbS0z1oXsUtzsgEXEnqDMBwn5Uu8BRYw6VpObXVQhV5e/qMTPKPPe8/axCs0WRrs1yA8cFVAC9eSpdMbble+Cz6h5C+NLVHVm7tLGyNpjeBpgGTmsfxupvv4wtkcpFGp5kRHj'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778483030, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:s-28KCHKFV2PW8G60RY67X|1748243028179&t:1748243030094&adb:adblk_no'}, {'domain': '.amazon.com', 'expirationDate': 1779779051.563287, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802977.837895, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1779779051.563317, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}], [{'domain': '.amazon.com', 'expirationDate': 1779779133.650745, 'hostOnly': False, 'httpOnly': False, 'name': 'ubid-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '133-7191575-7475824'}, {'domain': '.amazon.com', 'expirationDate': 1779779119.977571, 'hostOnly': False, 'httpOnly': False, 'name': 'session-token', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'exvqEGN+K9qT9LbO8WCb2f5eb1ciw/e1Vk69LmBSwds5r5SctvLXHTqeARNGQsi2Kx7FW0uqFHOiQLbn4pdCwGGRyEPT7GQP8UCnbKXG4XO3L/isrpjijmZxxiNg92wkpbIjtieIxIxvTyDYJh+pOw0pXjaGb094S5/jrKh1UOqNuk53Y1i5WD7cQGVWndMOholXeXh5soKbzgByJJ5nrYHU4hsL9ZoivXKSzFsMfNUpyUAHAtmAyBCxsSf/35kUG6oeW1qJcl0kFzCG+0MxJn6Oa3upv7Wfg9AgXy3ybgeZE1hPt41pd6t80jRMieyfROLs3Dot2WRJIMkl7Gqe83t1seCUDRkQ'}, {'domain': '.amazon.com', 'expirationDate': 1779777796.409894, 'hostOnly': False, 'httpOnly': False, 'name': 'i18n-prefs', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'USD'}, {'domain': 'www.amazon.com', 'expirationDate': 1778483123, 'hostOnly': True, 'httpOnly': False, 'name': 'csm-hit', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': 'tb:s-MGKHZFQBBBP3C18HJ63D|1748243121705&t:1748243123333&adb:adblk_no'}, {'domain': '.amazon.com', 'expirationDate': 1779779133.650835, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id-time', 'path': '/', 'secure': False, 'session': False, 'storeId': None, 'value': '2082787201l'}, {'domain': '.amazon.com', 'expirationDate': 1782802977.837895, 'hostOnly': False, 'httpOnly': False, 'name': 'lc-main', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': 'en_US'}, {'domain': '.amazon.com', 'expirationDate': 1779779133.650862, 'hostOnly': False, 'httpOnly': False, 'name': 'session-id', 'path': '/', 'secure': True, 'session': False, 'storeId': None, 'value': '138-2865349-0613132'}, {'domain': '.amazon.com', 'hostOnly': False, 'httpOnly': False, 'name': 'skin', 'path': '/', 'secure': False, 'session': True, 'storeId': None, 'value': 'noskin'}] ] def start_requests(self): detail_thread = threading.Thread(target=self.handle_details, args=()) detail_thread.start() dao.clear_spider_work_queue(self.name) with sync_playwright() as p: headless = False if platform.system() == 'Windows' else True browser = p.chromium.launch(headless=headless) context = browser.new_context() page = context.new_page() self.handle_search(page=page) detail_thread.join() dao.update_spider_cmd_info(self.name, self.cmd_id, 'stop_time', time.time()) dao.update_spider_cmd_info(self.name, self.cmd_id, 'status', 'stop') return [] def handle_page(self, page): while True: try: # 定义实体，文件，文件和实体关系进行关联 entities = {} files = [] buy_product_temp_list = [] buy_product_img_temp_list = [] post_dict = dao.get_spider_work_queue_wait(self.name) if not post_dict: timeout_count = 1 max_timeout_count = 3 if timeout_count > max_timeout_count: utils.log_print(f"超过{max_timeout_count}次未获取到待处理用户，退出") break post_dict = json.loads(post_dict) url = post_dict.get("post_url") page.goto(url, wait_until="domcontentloaded", timeout=60000) for i in range(10): page.evaluate(f"window.scrollBy(0, {random.randint(1500, 1800)})") time.sleep(random.randint(1, 2)) soup = BeautifulSoup(page.content(), 'lxml') time.sleep(random.randint(3, 5)) # 添加商品详情链接1 item_links = soup.find_all('div', attrs={'data-csa-c-type': 'item'}) if item_links: for item in item_links: item_link_tag = item.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": url, "depth": 1, } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接2 item_links_2 = soup.find_all('div', attrs={'data-testid': True}) if item_links_2: for item in item_links_2: item_link_tag = item.find('a', attrs={'href': True, 'data-testid': 'product-card-link'}) if item_link_tag: item_link = item_link_tag['href'] if item_link: if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": url, "depth": 1, } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接3 item_links_3 = soup.find_all('ol') if item_links_3: for item in item_links_3: item_lis = item.find_all('li') if item_lis: for item_1 in item_lis: item_link_tag = item_1.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if item_link: if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": url, "depth": 1, } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接4 item_links = soup.find_all('div', attrs={'class': 'p13n-sc-uncoverable-faceout'}) if item_links: for item in item_links: item_link_tag = item.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": url } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) #url为/video/ 视频详情页 if '/video/' in url : break # 获取商品详情 good_id = utils.str_to_md5(url) good_name = '' goods_img_list = [] good_name_h1 = soup.find('h1', attrs={'id': 'title'}) # 获取商品名 if good_name_h1: good_name = good_name_h1.text.strip() if good_name == '': break good_brand = '' good_brand_link = '' brand_a = soup.find('a', attrs={'id': 'bylineInfo'}) # 获取商家 if brand_a: good_brand_link = brand_a.get('href') good_brand = brand_a.text.replace('Brand:', '').strip() if self.platform_url not in good_brand_link: good_brand_link = self.platform_url + good_brand_link # 获取商品评价 customer_reviews = '' customer_reviews_div = soup.find('div', attrs={'id': 'averageCustomerReviews'}) if customer_reviews_div: customer_span = customer_reviews_div.find('span', attrs={'class': 'a-icon-alt'}) if customer_span: customer_reviews = customer_span.text.strip() # 获取商品评价数量 customer_reviews_count = '' customer_reviews_count_div = soup.find('div', attrs={'id': 'averageCustomerReviews'}) if customer_reviews_count_div: customer_reviews_count_span = customer_reviews_count_div.find('span', attrs={'id': 'acrCustomerReviewText'}) if customer_reviews_count_span: customer_reviews_count = customer_reviews_count_span.text.strip() # 获取商品价格 good_price = '' good_price_span = soup.find('span', attrs={'class': 'aok-offscreen'}) if good_price_span: good_price = good_price_span.text.strip() if 'with' in good_price: good_price = good_price.split('with')[0].strip() # 获取商品折扣 good_discount = '' good_discount_span = soup.find('span', attrs={ 'class': 'a-size-medium a-color-base apex-savings-text aok-align-center savingsPercentage'}) if good_discount_span: good_discount = good_discount_span.text.strip() # 获取商品原价 good_original_price = '' good_original_price_span = soup.find('span', attrs={'class': 'a-price a-text-price'}) if good_original_price_span: good_original_price_span_span = good_original_price_span.find('span') if good_original_price_span_span: good_original_price = good_original_price_span_span.text.strip() # 获取商品税 good_tax = '' good_tax_span = soup.find('span', attrs={'id': 'ifd_price', 'class': 'a-size-base'}) if good_tax_span: good_tax = good_tax_span.text.strip() #近期销量 recent_sales= '' good_tax_span = soup.find('div', attrs={'class': 'a-section a-spacing-micro social-proofing-faceout'}) if good_tax_span: good_tax = good_tax_span.text.strip() # 获取商品参数 good_arguments = '' good_arguments_tag = soup.find('table', class_='a-normal a-spacing-micro') data = {} if good_arguments_tag: good_trs = good_arguments_tag.find_all('tr') if len(good_trs) > 0: for row in good_trs: key_span = row.find('span', class_='a-size-base a-text-bold') value_span = row.find('span', class_='a-size-base po-break-word') if key_span and value_span: # 确保 key 和 value 都存在 key = key_span.text.strip() value = value_span.text.strip() data[key] = value good_arguments = json.dumps(data, ensure_ascii=False, indent=4) # 近期销量 recent_sales = '' recent_sales_tag = soup.find('div', attrs={'class': 'a-section a-spacing-micro social-proofing-faceout'}) if recent_sales_tag: recent_sales = recent_sales_tag.text.strip() # 获取商品描述(产品详情，产品介绍，产品描述三合一) good_describe = '' des_div = soup.find('div', attrs={'id': 'productDescription'}) if des_div: good_describe = des_div.text.strip() good_describe_div = soup.find('div', attrs={'id': 'feature-bullets'}) if good_describe_div: good_describe_ul = good_describe_div.find('ul') if good_describe_ul: good_describe_lis = good_describe_ul.find_all('li') if len(good_describe_lis) > 0: for li in good_describe_lis: good_describe += li.text.strip() # 使用CSS选择器匹配同时包含 a-unordered-list 和 a-vertical 的 ul 元素 uls = soup.select('ul.a-unordered-list.a-vertical') if len(uls) > 0: for ul in uls: lis = ul.find_all('li') if len(lis) > 0: for li in lis: good_describe += li.text.strip() # 定位到目标表格 table = soup.find('table', id='productDetails_detailBullets_sections1') # 初始化结果字符串 result_lines = [] if table: rows = table.find_all('tr') for row in rows: th = row.find('th') td = row.find('td') if th and td: key = th.get_text(strip=True) value = td.get_text(strip=True) # 特殊处理 Customer Reviews 和 Best Sellers Rank 等复杂结构 if key == 'Customer Reviews': rating_span = td.find('span', id='acrPopover') rating = rating_span.get('title') if rating_span else '' review_link = td.find('a', id='acrCustomerReviewLink') reviews_count = review_link.text.strip() if review_link else '' value = f'{rating}, {reviews_count}' if rating or reviews_count else '' elif key == 'Best Sellers Rank': lis = td.select('li span.a-list-item') best_sellers = [li.get_text(strip=True) for li in lis] value = ', '.join(best_sellers) if best_sellers else '' # 拼接到结果中 result_lines.append(f"{key}: {value}") # 将列表转换为字符串，每项换行 result = "\n".join(result_lines) good_describe = good_describe + result # 获取商品类型 good_type = '' good_type_ul = soup.find('ul', class_='a-unordered-list a-horizontal a-size-small') if good_type_ul: good_type = good_type_ul.text.strip().replace('\n', '') good_main_img = '' good_main_img_id = '' good_main_img_div = soup.find('div', attrs={'id': 'altImages'}) if good_main_img_div: img_list = good_main_img_div.find_all('img', attrs={'href': True}) if len(img_list) > 0: good_main_img = img_list[0]['src'] good_main_img_id = good_id + '.jpg' good_img_list = [] good_img_div = soup.find('div', attrs={'id': 'altImages'}) good_img_links = good_img_div.find_all('img', attrs={'href': True}) for img in good_img_links: good_img_link = img['src'] if '.gif' not in good_img_link: good_img_list.append(good_img_link) # 获取商品颜色 good_color_list = [] good_color_ul = soup.find('ul', class_='a-unordered-list a-nostyle a-button-list a-declarative a-button-toggle-group a-horizontal') if good_color_ul: good_color_lis = good_color_ul.find_all('li') if len(good_color_lis) > 0: for li in good_color_lis: good_color = li.find('img', attrs={'alt': True}) if good_color: good_color = good_color['alt'] good_color_list.append(good_color) good_color = li.find('img', attrs={'href': True}) if good_color: good_color_href = good_color['href'] goods_img_list.append(good_color_href) # 获取商品尺寸 good_size_list = [] good_size_select = soup.find('select', attrs={'id': 'native_dropdown_selected_size_name'}) if good_size_select: good_size_options = good_size_select.find_all('option', attrs={'data-a-html-content': True}) if len(good_size_options) > 0: for option in good_size_options: good_size = option.text.strip() good_size_list.append(good_size) good = BuyProduct(product_url=url, product_name=good_name, product_seller_name=good_brand, product_seller_url=good_brand_link, recent_sales=recent_sales, product_state='在售', overall_rating=customer_reviews, rating_count=customer_reviews_count, discount=good_discount, original_price=good_original_price, current_price=good_price, product_tax=good_tax, parameters=good_arguments, type=good_type, size=str(good_size_list), color=str(good_color_list), description=good_describe, country=self.country_pattern ) img_ul = soup.find('ul', class_='a-unordered-list a-nostyle a-horizontal list maintain-height') if img_ul: img_lis = img_ul.find_all('li') if len(img_lis) > 0: for li in img_lis: img_link = li.find('img', attrs={'src': True}) if img_link: img_link = img_link['src'] if '.gif' not in img_link: goods_img_list.append(img_link) img_div = soup.find('div', attrs={'id': 'Desktop-Detailed-Evaluation-Zone'}) if img_div: img_links = img_div.find_all('img', attrs={'src': True}) if len(img_links) > 0: for img in img_links: img_link = img['src'] if '.gif' not in img_link: goods_img_list.append(img_link) if len(goods_img_list) > 0: for img_link in goods_img_list[:5]: file_ext = '.jpg' if '.mp4' in img_link: file_ext = '.mp4' comment_img_id = utils.str_to_md5(img_link) comment_file = File( file_url=img_link, file_type='img', origin_url=img_link, file_ext=file_ext, name=f'{comment_img_id}{file_ext}', src_col='buy_products', src_id=utils.str_to_md5(url), title=good_name, src_url=url, ) buy_product_img_temp_list.append(comment_file) file_entities = {"files": buy_product_img_temp_list} file_product = CommonModel(spider=self, big_class=BigClass.BUY, entities=file_entities) file_product.save_img_file(img_url=img_link) # 保存图片,清空临时列表 buy_product_img_temp_list = [] buy_product_temp_list.append(good) product_entities = {"buy_products": buy_product_temp_list} file_product = CommonModel(spider=self, big_class=BigClass.BUY, entities=product_entities) file_product.save_file() time.sleep(random.randint(5, 10)) except Exception as e: utils.log_print(f'解析商品详情页失败：{traceback.format_exc()}') continue break def handle_details(self): with sync_playwright() as p: headless = False if platform.system() == 'Windows' else True browser = p.chromium.launch(headless=headless) context = browser.new_context() cookies = random.choice(self.cookies_list) context.add_cookies(cookies) page = context.new_page() page.set_viewport_size({"width": 1920, "height": 1080}) while True: time.sleep(random.randint(3, 5)) try: post_dict = dao.get_spider_work_queue_wait(self.name, 30) if not post_dict: timeout_count = 1 max_timeout_count = 3 if timeout_count > max_timeout_count: utils.log_print(f"超过{max_timeout_count}次未获取到待处理用户，退出") break utils.log_print(f"未获取到待处理商品， timeout_count: {timeout_count}") continue post_dict = json.loads(post_dict) publish_url = post_dict.get("post_url") self.handle_page(page) except Exception as e: utils.log_print(f"解析publish_url: {publish_url}详情出错： {e}") traceback.print_exc() def handle_search(self, page): page.goto("https://www.amazon.com/Best-Sellers/zgbs/ref=zg_bs_tab_bs", wait_until='domcontentloaded', timeout=60000) time.sleep(random.randint(5, 10)) html = page.content() soup = BeautifulSoup(html, 'lxml') # 添加商品详情链接1 # div class p13n-sc-uncoverable-faceout item_links = soup.find_all('div', attrs={'data-csa-c-type': 'item'}) if item_links: for item in item_links: item_link_tag = item.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": None, "depth": 1, } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接2 item_links_2 = soup.find_all('div', attrs={'data-testid': True}) if item_links_2: for item in item_links_2: item_link_tag = item.find('a', attrs={'href': True, 'data-testid': 'product-card-link'}) if item_link_tag: item_link = item_link_tag['href'] if item_link: if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": None } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接3 item_links_3 = soup.find_all('ol') if item_links_3: for item in item_links_3: item_lis = item.find_all('li') if item_lis: for item_1 in item_lis: item_link_tag = item_1.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if item_link: if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": None } dao.put_spider_work_queue(self.name, json.dumps(post_dict)) # 添加商品详情链接4 item_links = soup.find_all('div', attrs={'class': 'p13n-sc-uncoverable-faceout'}) if item_links: for item in item_links: item_link_tag = item.find('a', attrs={'href': True}) if item_link_tag: item_link = item_link_tag['href'] if 'http' not in item_link: item_link = self.channel_url + item_link post_dict = { "post_url": item_link, "parent_url": None } dao.put_spider_work_queue(self.name, json.dumps(post_dict))