编辑采集器       终止采集器

属性	值	属性	值
名称	us_mashable_news_history	创建时间	2025-04-02 10:10:02
标题	美国-Mashable历史---(US-2-采)(zxc)	更新时间	-
状态	running	开始时间	2025-07-05 06:45:16
停止类型		结束时间	-
信号		运行时长	60.2小时
进程ID	669	进程名称	Process-17
作业ID	b4a1141f-dc3f-47b2-a45f-6bf430cf9097	作业目录	records/jobs/b4a1141f-dc3f-47b2-a45f-6bf430cf9097
日志文件	records/logs/b4a1141f-dc3f-47b2-a45f-6bf430cf9097.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_mashable_news_history_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
aae5dd74-e541-4dfb-89d3-7d1e452f83b6	2025-07-04 00:26:26	-	90.6小时	3858	Process-13	查看日志	删除记录
删除最后 选择条数 条历史作业 确定

代码

import json from urllib.parse import urlparse, parse_qs import bs4 import requests from bs4 import BeautifulSoup from scrapy.http import Response import utils from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider import urllib.parse import re from spider.spiders.common.helper import update_url class UsMashableNewsHistorySpider(BaseQueueListToDetailsSpider): name = 'us_mashable_news_history' desc = '美国-Mashable历史---(US-2-采)(zxc)' net_name = 'Mashable,马沙布新闻,美国' net_url = 'mashable.com' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://mashable.com/' channel_desc = '马沙布新闻>新闻列表' max_page = 1000000 now_page = 0 from_reached_url = True host_key = 'mashable.com' exclude_urls = [] start_urls = [ # { # 'url': 'https://mashable.com/tech?page=2', # 'max_page': 1858, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'tech', # 'channel_url': 'https://mashable.com/tech', # 'channel_desc': 'mashable>tech' # } # , # { # 'url': 'https://mashable.com/science?page=2', # 'max_page': 239, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'science', # 'channel_url': 'https://mashable.com/science', # 'channel_desc': 'mashable>science' # } # , # { # 'url': 'https://mashable.com/life?page=2', # 'max_page': 1868, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'life', # 'channel_url': 'https://mashable.com/life', # 'channel_desc': 'mashable>life' # } # , { 'url': 'https://mashable.com/category/social-good?page=2', 'max_page': 143, 'language': 'en', 'nation_category': 1, 'channel_name': 'social-good', 'channel_url': 'https://mashable.com/category/social-good', 'channel_desc': 'mashable>social-good' } # , # { # 'url': 'https://mashable.com/entertainment?page=2', # 'max_page': 1539, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'entertainment', # 'channel_url': 'https://mashable.com/entertainment', # 'channel_desc': 'mashable>entertainment' # } # , # { # 'url': 'https://mashable.com/deals?page=2', # 'max_page': 918, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'deals', # 'channel_url': 'https://mashable.com/deals', # 'channel_desc': 'mashable>deals' # } # , # { # 'url': 'https://mashable.com/shopping?page=2', # 'max_page': 970, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'shopping', # 'channel_url': 'https://mashable.com/shopping', # 'channel_desc': 'mashable>shopping' # } ] def build_next_list_url(self, response, list_url_item): if response: updated_url = '' use_url = response.url print('use_url') print(use_url) if 'page' in use_url: # 使用正则表达式提取 page 参数的数字 match = re.search(r'page=(\d+)', use_url) if match: last_number = int(match.group(1)) # 获取最后的数字 new_number = last_number + 1 # 原有数字加 1 # 生成新的 URL updated_url = use_url.replace(f'page={last_number}', f'page={new_number}') else: updated_url = use_url + '?page=2' list_url_item[ 'use_url'] = updated_url return list_url_item def build_detail_urls(self, response: Response): detail_url_list = [] if response: soup = bs4.BeautifulSoup(response.text,'lxml') head_div = soup.find('div',attrs={'data-module':'content-list'}) if head_div: print('找到head_div') articles = head_div.find_all('div', class_='w-full',recursive=False) if len(articles)>0: for article in articles: if article.find('a', attrs={'href': True}): detail_url = article.find('a', attrs={'href': True})['href'] if 'https://mashable.com' not in detail_url: detail_url = 'https://mashable.com'+detail_url detail_url_list.append(detail_url) return detail_url_list def get_img(soup: BeautifulSoup): img_url = '' try: content_div = soup.find('section', attrs={'data-ga-module':'content_body'}) if content_div: img_url = content_div.find('img')['src'] except Exception as e: utils.log_error(f'get img fail: {e}') return img_url img_href_rules = [ { 'op': 'func', 'name': get_img } ] title_rules = [ { 'op': 'find', 'tag': 'h1' }, { 'op': 'text' } ] # propos -> pageProps -> globalContent -> promo_items -> basic -> credits -> by -> name def get_author(soup: BeautifulSoup): name = '' try: title_str = soup.find('a',class_ = 'underline-link').text.strip().replace('\n', '').replace('\t', '') if title_str: if '-' in title_str: name_str = title_str.split('-')[0].strip() if 'By' in name_str: name = name_str.replace('By', '') if 'by' in name_str: name = name_str.replace('by', '') name = title_str print(name) except Exception as e: utils.log_error(f'get img fail: {e}') return name author_rules = [ { 'op': 'func', 'name': get_author } ] def get_time(soup: BeautifulSoup): time = '' try: time = soup.find('time',attrs={'datetime':True}).text if time: time = time.strip().replace('\n','').replace('\t','') print(time) except Exception as e: utils.log_error(f'get img fail: {e}') return time time_rules = [ { 'op': 'func', 'name': get_time } ] # propos -> pageProps -> globalContent -> content_elements -> {url} / {content...} (img /content) def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('article', attrs={'id':'article'}) if content_div: content_ps = content_div.find_all('p') if len(content_ps)>0: for p in content_ps: content = content +'\n\n'+ p.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ]