编辑采集器       启动采集器

属性	值	属性	值
名称	uk_metro_news	创建时间	2025-04-02 10:10:02
标题	英国-地铁报(zxc1.0)	更新时间	-
状态	stop	开始时间	2025-07-05 20:58:45
停止类型	finished	结束时间	2025-07-05 20:58:45
信号		运行时长	0秒
进程ID	301	进程名称	Process-8
作业ID	5149cbe6-8b97-495a-9412-3be0f174afa4	作业目录	records/jobs/5149cbe6-8b97-495a-9412-3be0f174afa4
日志文件	records/logs/5149cbe6-8b97-495a-9412-3be0f174afa4.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	uk_metro_news_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import json from urllib.parse import urlparse, parse_qs import bs4 from bs4 import BeautifulSoup from scrapy.http import Response import utils from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider import urllib.parse import re from spider.spiders.common.helper import update_url class UkMetroNewsSpider(BaseQueueListToDetailsSpider): name = 'uk_metro_news' desc = '英国-地铁报(zxc1.0)' net_name = 'Metro,地铁报,英国' net_url = 'metro.co.uk' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://metro.co.uk' channel_desc = '地铁报>新闻列表' max_page = 100000 now_page = 0 from_reached_url = False host_key = 'metro.co.uk' exclude_urls = [] start_urls = [ { 'url': 'https://metro.co.uk/wp-json/wp/v2/posts/?per_page=23&before=2024-12-26T01%3A33%3A10%2B00%3A00&after=2024-09-27T01%3A33%3A10%2B00%3A00&_fields=metro_content_item%2Cfeatured_media_sizes&metro_content_item_rendered=true&metro_content_item_module=latest&metro-latest-load-more=true&categories=8199&page=1', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'uk', 'channel_url': 'https://metro.co.uk/news/uk/', 'channel_desc': 'metro>uk' } , { 'url': 'https://metro.co.uk/wp-json/wp/v2/posts/?per_page=23&before=2024-12-26T01%3A36%3A03%2B00%3A00&after=2024-09-27T01%3A36%3A03%2B00%3A00&_fields=metro_content_item%2Cfeatured_media_sizes&metro_content_item_rendered=true&metro_content_item_module=latest&metro-latest-load-more=true&categories=472659319&page=1', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'us', 'channel_url': 'https://metro.co.uk/news/us', 'channel_desc': 'metro>us' },{ 'url': 'https://metro.co.uk/wp-json/wp/v2/posts/?per_page=23&before=2024-12-26T01%3A32%3A59%2B00%3A00&after=2024-09-27T01%3A32%3A59%2B00%3A00&_fields=metro_content_item%2Cfeatured_media_sizes&metro_content_item_rendered=true&metro_content_item_module=latest&metro-latest-load-more=true&categories=1857&page=1', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'world', 'channel_url': 'https://metro.co.uk/news/world/', 'channel_desc': 'metro>world' },{ 'url': 'https://metro.co.uk/wp-json/wp/v2/posts/?per_page=23&before=2024-12-26T01%3A39%3A48%2B00%3A00&after=2024-09-27T01%3A39%3A48%2B00%3A00&_fields=metro_content_item%2Cfeatured_media_sizes&metro_content_item_rendered=true&metro_content_item_module=latest&metro-latest-load-more=true&categories=318&page=1', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'tech', 'channel_url': 'https://metro.co.uk/news/tech/', 'channel_desc': 'metro>tech' },{ 'url': 'https://metro.co.uk/wp-json/wp/v2/posts/?per_page=23&before=2024-12-26T01%3A40%3A47%2B00%3A00&after=2024-09-27T01%3A40%3A47%2B00%3A00&_fields=metro_content_item%2Cfeatured_media_sizes&metro_content_item_rendered=true&metro_content_item_module=latest&metro-latest-load-more=true&categories=670350652&page=1', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'advice', 'channel_url': 'https://metro.co.uk/news/politics/', 'channel_desc': 'metro>politics' } ] #metro_content_item -> source -> link(value) def build_next_list_url(self, response, list_url_item): if response: soup = BeautifulSoup(response.text, 'lxml') nav = soup.find('nav',class_ = 'topic-stories-list__pagination') a_list = nav.find_all('a') for a in a_list: if a.text == 'Next →': print('找到a标签') a_url = a['href'] if 'https://slate.com/' in a_url: list_url_item['use_url'] = a_url print(f'a_url:{a_url}') else: new_url = 'https://slate.com' + a_url list_url_item['use_url'] = new_url print(f'new_url:{new_url}') break return list_url_item def build_detail_urls(self, response: Response): detail_url_list = [] if response: soup = bs4.BeautifulSoup(response.text, 'lxml') head_dad_div = soup.find('div', class_='layout-teaser-list__main-content') head_div = head_dad_div.find('div',recursive=False) if head_div: heads_a = head_div.find_all('a', attrs={'href': True},recursive=False) if len(heads_a)> 0: for a in heads_a: if 'https://slate.com' not in a['href']: url = a['href'] print(f'a[href]:{url}') a['href'] = 'https://slate.com' + a['href'] detail_url_list.append(a['href']) else: url = a['href'] print(f'a[href]:{url}') detail_url_list.append(url) return detail_url_list def get_img(soup: BeautifulSoup): img_url = '' try: content_img = soup.find('img', atrrs={'src': True}) if content_img: img_url = content_img['href'] except Exception as e: utils.log_error(f'get img fail: {e}') return img_url img_href_rules = [ { 'op': 'func', 'name': get_img } ] title_rules = [ { 'op': 'find', 'tag': 'h1' }, { 'op': 'text' } ] # propos -> pageProps -> globalContent -> promo_items -> basic -> credits -> by -> name def get_author(soup: BeautifulSoup): name = '' try: name_div = soup.find('div', class_='article__byline') if name_div: name_span = name_div.find('span',attrs={'itemprop':'author'}) if name_span: name = name_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return name author_rules = [ { 'op': 'func', 'name': get_author } ] def get_time(soup: BeautifulSoup): time = '' try: time_span = soup.find('time',class_ = 'article__dateline', attrs={'content': True}) if time_span: time = time_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return time time_rules = [ { 'op': 'func', 'name': get_time } ] # propos -> pageProps -> globalContent -> content_elements -> {url} / {content...} (img /content) def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('div', attrs={'class': 'article__content'}) if content_div: content_ps = content_div.find_all('p') if len(content_ps): for p in content_ps: content = content + '\n\n' + p.text.strip().replace('\n', '').replace('\t', '') video_div = soup.find('div', class_='podcast-episode__main-bottom') if video_div: video_ps = video_div.find_all('p') if len(video_ps) > 0: for p in video_ps: content = content + '\n\n' + p.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ]