编辑采集器       终止采集器

属性	值	属性	值
名称	uk_sky_news	创建时间	2025-04-02 10:10:02
标题	英国-天空新闻(zxc1.0)	更新时间	-
状态	running	开始时间	2025-07-05 10:58:16
停止类型		结束时间	-
信号		运行时长	58.8小时
进程ID	730	进程名称	Process-23
作业ID	20253a54-a5aa-4b7e-8d02-bbc44c7711f9	作业目录	records/jobs/20253a54-a5aa-4b7e-8d02-bbc44c7711f9
日志文件	records/logs/20253a54-a5aa-4b7e-8d02-bbc44c7711f9.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	uk_sky_news_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import json from urllib.parse import urlparse, parse_qs import bs4 from bs4 import BeautifulSoup from scrapy.http import Response import utils from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider import urllib.parse import re from spider.spiders.common.helper import update_url class UkSkyNewsSpider(BaseQueueListToDetailsSpider): name = 'uk_sky_news' desc = '英国-天空新闻(zxc1.0)' net_name = 'Sky News,天空新闻,英国' net_url = 'news.sky.com' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://news.sky.com/' channel_desc = '天空新闻>新闻列表' max_page = 100000 now_page = 0 from_reached_url = False host_key = 'news.sky.com' exclude_urls = [] start_urls = [ { 'url': 'https://news.sky.com/uk', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'uk', 'channel_url': 'https://news.sky.com/uk', 'channel_desc': 'sky>uk' },{ 'url': 'https://news.sky.com/politics', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'politics', 'channel_url': 'https://news.sky.com/politics', 'channel_desc': 'sky>politics' },{ 'url': 'https://news.sky.com/world', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'world', 'channel_url': 'https://news.sky.com/world', 'channel_desc': 'sky>world' },{ 'url': 'https://news.sky.com/us', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'us', 'channel_url': 'https://news.sky.com/us', 'channel_desc': 'sky>us' },{ 'url': 'https://news.sky.com/money', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'money', 'channel_url': 'https://news.sky.com/money', 'channel_desc': 'sky>money' },{ 'url': 'https://news.sky.com/science-climate-tech', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'science-climate-tech', 'channel_url': 'https://news.sky.com/science-climate-tech', 'channel_desc': 'sky>science-climate-tech' },{ 'url': 'https://news.sky.com/entertainment', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'entertainment', 'channel_url': 'https://news.sky.com/entertainment', 'channel_desc': 'sky>entertainment' },{ 'url': 'https://news.sky.com/programmes', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'programmes', 'channel_url': 'https://news.sky.com/programmes', 'channel_desc': 'sky>programmes' },{ 'url': 'https://news.sky.com/analysis', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'analysis', 'channel_url': 'https://news.sky.com/analysis', 'channel_desc': 'sky>analysis' } ,{ 'url': 'https://news.sky.com/data-and-forensics', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'data-and-forensics', 'channel_url': 'https://news.sky.com/data-and-forensics', 'channel_desc': 'sky>data-and-forensics' },{ 'url': 'https://news.sky.com/offbeat', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'offbeat', 'channel_url': 'https://news.sky.com/offbeat', 'channel_desc': 'sky>offbeat' } ] #未编写下一页的逻辑代码 def build_next_list_url(self, response, list_url_item): if response: soup = BeautifulSoup(response.text, 'lxml') nav = soup.find('nav',class_ = 'topic-stories-list__pagination') a_list = nav.find_all('a') for a in a_list: if a.text == 'Next →': print('找到a标签') a_url = a['href'] if 'https://slate.com/' in a_url: list_url_item['use_url'] = a_url print(f'a_url:{a_url}') else: new_url = 'https://slate.com' + a_url list_url_item['use_url'] = new_url print(f'new_url:{new_url}') break return list_url_item def build_detail_urls(self, response: Response): detail_url_list = [] if response: soup = bs4.BeautifulSoup(response.text, 'lxml') a_list = soup.find_all('a',class_ = 'ui-story-headline',attrs={'href':True}) if len(a_list)>0: for a in a_list: if 'https://news.sky.com' not in a['href']: url = 'https://news.sky.com' + a['href'] print(f'a[href]:{url}') a['href'] = 'https://news.sky.com' + a['href'] detail_url_list.append(a['href']) else: url = a['href'] print(f'a[href]:{url}') detail_url_list.append(url) return detail_url_list def get_img(soup: BeautifulSoup): img_urls = [] try: head_div = soup.find('div',class_ = 'sdc-article-image__wrapper') img = head_div.find('img', attrs={'src':True}) img_urls.append(img['src']) except Exception as e: utils.log_error(f'get img fail: {e}') return img_urls img_href_rules = [ { 'op': 'func', 'name': get_img } ] title_rules = [ { 'op': 'find', 'tag': 'h1' }, { 'op': 'text' } ] # propos -> pageProps -> globalContent -> promo_items -> basic -> credits -> by -> name def get_author(soup: BeautifulSoup): name = '' try: name_span = soup.find('span', class_='sdc-article-author__name') if name_span : if name_span.find('a'): name = name_span.find('a').text.strip().replace('\n', '').replace('\t', '') else: name = name_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return name author_rules = [ { 'op': 'func', 'name': get_author } ] def get_time(soup: BeautifulSoup): time = '' try: time_span = soup.find('p',class_ = 'sdc-article-date__date-time') if time_span: time = time_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return time time_rules = [ { 'op': 'func', 'name': get_time } ] # propos -> pageProps -> globalContent -> content_elements -> {url} / {content...} (img /content) def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('div', attrs={'data-component-name': 'ui-article-body'}) if content_div: p_list = content_div.find_all('p') if len(p_list)>0: for p in p_list: content_p = p.text.strip().replace('\n', '').replace('\t', '') content = content + '\n\n' + content_p except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ]