编辑采集器       启动采集器

属性	值	属性	值
名称	us_slate_news_history	创建时间	2025-04-02 10:10:02
标题	美国-Slate新闻历史---(US-2-采)(zxc)	更新时间	-
状态	stop	开始时间	2025-06-29 01:26:35
停止类型	finished	结束时间	2025-06-29 01:29:36
信号		运行时长	3.0分钟
进程ID	2345060	进程名称	Process-2
作业ID	8056cde3-d018-4d98-b445-c0922ab02647	作业目录	records/jobs/8056cde3-d018-4d98-b445-c0922ab02647
日志文件	records/logs/8056cde3-d018-4d98-b445-c0922ab02647.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_slate_news_history_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import json from urllib.parse import urlparse, parse_qs import bs4 from bs4 import BeautifulSoup from scrapy.http import Response import utils from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider import urllib.parse import re from spider.spiders.common.helper import update_url class UsSlateNewsHistorySpider(BaseQueueListToDetailsSpider): name = 'us_slate_news_history' desc = '美国-Slate新闻历史---(US-2-采)(zxc)' net_name = 'Slate,Slate新闻,美国' net_url = 'slate.com' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://slate.com/' channel_desc = 'Slate新闻>新闻列表' max_page = 100000 now_page = 0 from_reached_url = True host_key = 'slate.com' exclude_urls = [] start_urls = [ { 'url': 'https://slate.com/news-and-politics/', 'max_page': 2000, 'language': 'en', 'nation_category': 1, 'channel_name': 'news-and-politics', 'channel_url': 'https://slate.com/news-and-politics/', 'channel_desc': 'slate>news-and-politics' } # , # { # 'url': 'https://slate.com/culture', # 'max_page': 2000, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'culture', # 'channel_url': 'https://slate.com/culture', # 'channel_desc': 'slate>culture' # } # , # { # 'url': 'https://slate.com/technology', # 'max_page': 2000, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'technology', # 'channel_url': 'https://slate.com/technology', # 'channel_desc': 'slate>technology' # },{ # 'url': 'https://slate.com/business', # 'max_page': 2000, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'business', # 'channel_url': 'https://slate.com/business', # 'channel_desc': 'slate>business' # },{ # 'url': 'https://slate.com/life', # 'max_page': 2000, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'life', # 'channel_url': 'https://slate.com/life', # 'channel_desc': 'slate>life' # },{ # 'url': 'https://slate.com/advice', # 'max_page': 2000, # 'language': 'en', # 'nation_category': 1, # 'channel_name': 'advice', # 'channel_url': 'https://slate.com/advice', # 'channel_desc': 'slate>advice' # } ] def build_next_list_url(self, response, list_url_item): if response: soup = BeautifulSoup(response.text, 'lxml') nav = soup.find('nav',class_ = 'topic-stories-list__pagination') a_list = nav.find_all('a') for a in a_list: if a.text == 'Next →': print('找到a标签') a_url = a['href'] if 'https://slate.com/' in a_url: list_url_item['use_url'] = a_url print(f'a_url:{a_url}') else: new_url = 'https://slate.com' + a_url list_url_item['use_url'] = new_url print(f'new_url:{new_url}') break return list_url_item def build_detail_urls(self, response: Response): detail_url_list = [] if response: soup = bs4.BeautifulSoup(response.text, 'lxml') head_dad_div = soup.find('div', class_='layout-teaser-list__main-content') head_div = head_dad_div.find('div',recursive=False) if head_div: heads_a = head_div.find_all('a', attrs={'href': True},recursive=False) if len(heads_a)> 0: for a in heads_a: if 'https://slate.com' not in a['href']: url = a['href'] print(f'a[href]:{url}') a['href'] = 'https://slate.com' + a['href'] detail_url_list.append(a['href']) else: url = a['href'] print(f'a[href]:{url}') detail_url_list.append(url) return detail_url_list def get_img(soup: BeautifulSoup): img_url = '' try: content_img = soup.find('img', atrrs={'src': True}) if content_img: img_url = content_img['href'] except Exception as e: utils.log_error(f'get img fail: {e}') return img_url img_href_rules = [ { 'op': 'func', 'name': get_img } ] title_rules = [ { 'op': 'find', 'tag': 'h1' }, { 'op': 'text' } ] # propos -> pageProps -> globalContent -> promo_items -> basic -> credits -> by -> name def get_author(soup: BeautifulSoup): name = '' try: name_div = soup.find('div', class_='article__byline') if name_div: name_span = name_div.find('span',attrs={'itemprop':'author'}) if name_span: name = name_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return name author_rules = [ { 'op': 'func', 'name': get_author } ] def get_time(soup: BeautifulSoup): time = '' try: time_span = soup.find('time',class_ = 'article__dateline', attrs={'content': True}) if time_span: time = time_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return time time_rules = [ { 'op': 'func', 'name': get_time } ] # propos -> pageProps -> globalContent -> content_elements -> {url} / {content...} (img /content) def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('div', attrs={'class': 'article__content'}) if content_div: content_ps = content_div.find_all('p') if len(content_ps): for p in content_ps: content = content + '\n\n' + p.text.strip().replace('\n', '').replace('\t', '') video_div = soup.find('div', class_='podcast-episode__main-bottom') if video_div: video_ps = video_div.find_all('p') if len(video_ps) > 0: for p in video_ps: content = content + '\n\n' + p.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ]