编辑采集器       终止采集器

属性	值	属性	值
名称	uk_the_times_news	创建时间	2025-04-02 10:10:02
标题	英国-泰晤士报(zxc1.0)	更新时间	-
状态	running	开始时间	2025-07-04 00:26:34
停止类型		结束时间	-
信号		运行时长	89.2小时
进程ID	3871	进程名称	Process-17
作业ID	984a3088-2430-4abb-8420-98ca8d618cf7	作业目录	records/jobs/984a3088-2430-4abb-8420-98ca8d618cf7
日志文件	records/logs/984a3088-2430-4abb-8420-98ca8d618cf7.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	uk_the_times_news_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import json from urllib.parse import urlparse, parse_qs import bs4 from bs4 import BeautifulSoup from scrapy.http import Response import utils from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider import urllib.parse import re from spider.spiders.common.helper import update_url class UkTheTimesNewsSpider(BaseQueueListToDetailsSpider): name = 'uk_the_times_news' desc = '英国-泰晤士报(zxc1.0)' net_name = 'The Times,泰晤士报,英国' net_url = 'thetimes.com' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://www.thetimes.com/' channel_desc = '泰晤士报>新闻列表' max_page = 100000 now_page = 0 from_reached_url = False host_key = 'thetimes.com' exclude_urls = [] start_urls = [ { 'url': 'https://www.thetimes.com/uk/politics', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'politics', 'channel_url': 'https://www.thetimes.com/uk/politics', 'channel_desc': 'thetimes>uk>politics' } ,{ 'url': 'https://www.thetimes.com/uk/healthcare', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'healthcare', 'channel_url': 'https://www.thetimes.com/uk/healthcare', 'channel_desc': 'thetimes>uk>healthcare' },{ 'url': 'https://www.thetimes.com/uk/education', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'education', 'channel_url': 'https://www.thetimes.com/uk/education', 'channel_desc': 'thetimes>uk>education' },{ 'url': 'https://www.thetimes.com/uk/crime', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'crime', 'channel_url': 'https://www.thetimes.com/uk/crime', 'channel_desc': 'thetimes>uk>crime' },{ 'url': 'https://www.thetimes.com/uk/defence', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'defence', 'channel_url': 'https://www.thetimes.com/uk/defence', 'channel_desc': 'thetimes>uk>defence' },{ 'url': 'https://www.thetimes.com/uk/scotland', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'scotland', 'channel_url': 'https://www.thetimes.com/uk/scotland', 'channel_desc': 'thetimes>uk>scotland' },{ 'url': 'https://www.thetimes.com/uk/obituaries', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'obituaries', 'channel_url': 'https://www.thetimes.com/uk/obituaries', 'channel_desc': 'thetimes>uk>obituaries' },{ 'url': 'https://www.thetimes.com/uk/royal-family', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'royal-family', 'channel_url': 'https://www.thetimes.com/uk/royal-family', 'channel_desc': 'thetimes>uk>royal-family' },{ 'url': 'https://www.thetimes.com/uk/science', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'science', 'channel_url': 'https://www.thetimes.com/uk/science', 'channel_desc': 'thetimes>uk>science' } ,{ 'url': 'https://www.thetimes.com/uk/environment', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'environment', 'channel_url': 'https://www.thetimes.com/uk/environment', 'channel_desc': 'thetimes>uk>environment' },{ 'url': 'https://www.thetimes.com/uk/law', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'culture', 'channel_url': 'https://www.thetimes.com/uk/law', 'channel_desc': 'thetimes>uk>law' },{ 'url': 'https://www.thetimes.com/world/us', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'us', 'channel_url': 'https://www.thetimes.com/world/us', 'channel_desc': 'thetimes>world>us' },{ 'url': 'https://www.thetimes.com/world/middle-east', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'middle-east', 'channel_url': 'https://www.thetimes.com/world/middle-east', 'channel_desc': 'thetimes>world>middle-east' },{ 'url': 'https://www.thetimes.com/world/russia-ukraine-war', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'russia-ukraine-war', 'channel_url': 'https://www.thetimes.com/world/russia-ukraine-war', 'channel_desc': 'thetimes>world>russia-ukraine-war' },{ 'url': 'https://www.thetimes.com/world/europe', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'europe', 'channel_url': 'https://www.thetimes.com/world/europe', 'channel_desc': 'thetimes>world>europe' },{ 'url': 'https://www.thetimes.com/world/ireland', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'ireland', 'channel_url': 'https://www.thetimes.com/world/ireland', 'channel_desc': 'thetimes>world>ireland' },{ 'url': 'https://www.thetimes.com/world/asia', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'asia', 'channel_url': 'https://www.thetimes.com/world/asia', 'channel_desc': 'thetimes>world>asia' },{ 'url': 'https://www.thetimes.com/world/latin-america', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'latin-america', 'channel_url': 'https://www.thetimes.com/world/latin-america', 'channel_desc': 'thetimes>world>latin-america' },{ 'url': 'https://www.thetimes.com/uk/politics', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'culture', 'channel_url': 'https://www.thetimes.com/uk/politics', 'channel_desc': 'thetimes>uk>politics' },{ 'url': 'https://www.thetimes.com/world/africa', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'africa', 'channel_url': 'https://www.thetimes.com/world/africa', 'channel_desc': 'thetimes>world>africa' },{ 'url': 'https://www.thetimes.com/world/australasia', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'australasia', 'channel_url': 'https://www.thetimes.com/world/australasia', 'channel_desc': 'thetimes>world>australasia' },{ 'url': 'https://www.thetimes.com/comment/columnists', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'columnists', 'channel_url': 'https://www.thetimes.com/comment/columnists', 'channel_desc': 'thetimes>comment>columnists' },{ 'url': 'https://www.thetimes.com/comment/the-times-view', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'the-times-view', 'channel_url': 'https://www.thetimes.com/comment/the-times-view', 'channel_desc': 'thetimes>comment>the-times-view' },{ 'url': 'https://www.thetimes.com/comment/letters-to-editor', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'letters-to-editor', 'channel_url': 'https://www.thetimes.com/comment/letters-to-editor', 'channel_desc': 'thetimes>comment>letters-to-editor' },{ 'url': 'https://www.thetimes.com/comment/cartoons', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'cartoons', 'channel_url': 'https://www.thetimes.com/comment/cartoons', 'channel_desc': 'thetimes>comment>cartoons' },{ 'url': 'https://www.thetimes.com/comment/universal-register', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'universal-register', 'channel_url': 'https://www.thetimes.com/comment/universal-register', 'channel_desc': 'thetimes>comment>universal-register' },{ 'url': 'https://www.thetimes.com/business-money/money', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'money', 'channel_url': 'https://www.thetimes.com/business-money/money', 'channel_desc': 'thetimes>business-money>money' },{ 'url': 'https://www.thetimes.com/business-money/economics', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'economics', 'channel_url': 'https://www.thetimes.com/business-money/economics', 'channel_desc': 'thetimes>business-money>economics' },{ 'url': 'https://www.thetimes.com/business-money/markets-tempus', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'markets-tempus', 'channel_url': 'https://www.thetimes.com/business-money/markets-tempus', 'channel_desc': 'thetimes>business-money>markets-tempus' } ,{ 'url': 'https://www.thetimes.com/business-money/technology', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'technology', 'channel_url': 'https://www.thetimes.com/business-money/technology', 'channel_desc': 'thetimes>business-money>technology' },{ 'url': 'https://www.thetimes.com/business-money/companies', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'companies', 'channel_url': 'https://www.thetimes.com/business-money/companies', 'channel_desc': 'thetimes>business-money>companies' },{ 'url': 'https://www.thetimes.com/business-money/energy', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'energy', 'channel_url': 'https://www.thetimes.com/business-money/energy', 'channel_desc': 'thetimes>business-money>energy' } ] #metro_content_item -> source -> link(value) def build_next_list_url(self, response, list_url_item): if response: url = response.url if 'page' in url: # 获取当前页码 current_page = url.split('=')[-1] next_page = int(current_page) + 1 left_url = url.split('=')[0] next_page_url = left_url + '=' + str(next_page) else: next_page_url = url + '?page=2' list_url_item['use_url'] = next_page_url return list_url_item def build_detail_urls(self, response: Response): detail_url_list = [] if response: soup = bs4.BeautifulSoup(response.text, 'lxml') divs = soup.find_all('div',attrs={'data-testid':'lead-article'}) if len(divs)>0: for div in divs: a = div.find('a', attrs={'href': True}) if a: detail_url = a['href'] if 'https://www.thetimes.com' not in detail_url: detail_url = 'https://www.thetimes.com' + detail_url detail_url_list.append(detail_url) else: detail_url_list.append(detail_url) return detail_url_list def get_img(soup: BeautifulSoup): img_url = '' try: main = soup.find('main') content_img = main.find('img', atrrs={'src': True}) if content_img: print('content_img:', content_img) img_url = content_img['href'] except Exception as e: utils.log_error(f'get img fail: {e}') return img_url img_href_rules = [ { 'op': 'func', 'name': get_img } ] title_rules = [ { 'op': 'find', 'tag': 'h1' }, { 'op': 'text' } ] # propos -> pageProps -> globalContent -> promo_items -> basic -> credits -> by -> name def get_author(soup: BeautifulSoup): name = '' try: name_a = soup.find('a', attrs={'class': 'tc-text-link__TcTextLink-sc-1voa8bp-0 text-link__LinkTextObj-xyehx2-0 vSslN'}) if name_a: name = name_a.text.strip() except Exception as e: utils.log_error(f'get img fail: {e}') return name author_rules = [ { 'op': 'func', 'name': get_author } ] def get_time(soup: BeautifulSoup): time = '' try: time_span = soup.find('time',attrs={'datetime': True}) if time_span: time = time_span.text.strip().replace('\n', '').replace('\t', '') except Exception as e: utils.log_error(f'get img fail: {e}') return time time_rules = [ { 'op': 'func', 'name': get_time } ] # propos -> pageProps -> globalContent -> content_elements -> {url} / {content...} (img /content) def get_content(soup: BeautifulSoup): content = '' try: main = soup.find('main') article = main.find('article') tag_div = article.find('div',recursive=False) ps = tag_div.find_all('p') if len(ps) > 0: print(len(ps)) for p in ps: word = p.text print(word) content = content + '\n\n' +word.strip().replace('\t','').replace('\n','') return content except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ]