编辑采集器       终止采集器

属性	值	属性	值
名称	us_usnews	创建时间	2025-04-02 10:10:02
标题	美国-USNews---(US-1-采)	更新时间	-
状态	running	开始时间	2025-07-05 11:46:16
停止类型		结束时间	-
信号		运行时长	56.1小时
进程ID	736	进程名称	Process-24
作业ID	4707a46a-022b-49b6-928e-36c417bafc7f	作业目录	records/jobs/4707a46a-022b-49b6-928e-36c417bafc7f
日志文件	records/logs/4707a46a-022b-49b6-928e-36c417bafc7f.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	us_usnews_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
4fdca2e3-d191-42f1-a750-f994c827d40b	2025-07-04 00:26:27	-	91.4小时	3861	Process-14	查看日志	删除记录
删除最后 选择条数 条历史作业 确定

代码

from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider from scrapy.http import Response from urllib.parse import urlparse, parse_qs import json import utils # us_usnews_spider.py class UsUsnewsSpider(BaseQueueListToDetailsSpider): name = 'us_usnews' desc = '美国-USNews---(US-1-采)' net_name = 'USNews,美国新闻,美国' net_url = 'www.usnews.com' language = 'en' nation_category = 1 channel_name = '新闻列表' channel_url = 'https://www.usnews.com' channel_desc = 'USNews>新闻列表' max_page = 1000 now_page = 0 from_reached_url = False host_key = 'usnews.com' exclude_urls = [] def custom_get_headers(self, request_url=None): headers = {} if request_url and 'renderer=json' in request_url: cookie_name = 'usn_visitor_id' cookie_value = utils.get_random_str(20) headers['Cookie'] = f'{cookie_name}={cookie_value}' headers['Accept-Language'] = f'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2' headers['Accept-Encoding'] = f'gzip, deflate, br, zstd' headers['User-Agent'] = f'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:130.0) Gecko/20100101 Firefox/130.0' headers['Host'] = f'www.usnews.com' headers['Referer'] = f'https://www.usnews.com/news' headers['Another-Header'] = f'value' return headers ''' https://www.usnews.com/news/national-news?offset=0&renderer=json https://www.usnews.com/news/politics-news?offset=0&renderer=json https://www.usnews.com/news/political-news?offset=0&renderer=json ''' start_urls = [ { 'url':'https://www.usnews.com/news/national-news?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'National News', 'channel_url': 'https://www.usnews.com/news/national-news', 'channel_desc': 'USNews>News>National News' }, { 'url':'https://www.usnews.com/news/politics-news?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'Politics News', 'channel_url': 'https://www.usnews.com/news/politics-news', 'channel_desc': 'USNews>News>Politics News' }, { 'url':'https://www.usnews.com/news/political-news?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'Political News', 'channel_url': 'https://www.usnews.com/news/political-news', 'channel_desc': 'USNews>News>Political News' }, { 'url':'https://www.usnews.com/news/world-news?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'World News', 'channel_url': 'https://www.usnews.com/news/world-news', 'channel_desc': 'USNews>News>World News' }, { 'url':'https://www.usnews.com/news/business?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'Business', 'channel_url': 'https://www.usnews.com/news/business', 'channel_desc': 'USNews>News>Business' }, { 'url':'https://www.usnews.com/science/news?renderer=json&offset=0', 'max_page': 1, 'language': 'en', 'nation_category': 1, 'channel_name': 'Science News', 'channel_url': 'https://www.usnews.com/science/news', 'channel_desc': 'USNews>News>Science News' } ] def get_list_url(self, channel, from_record): list_url = f'https://www.usnews.com/news/{channel}?' list_url = list_url + f'renderer=json' list_url = list_url + f'&offset={from_record}' return list_url def build_next_list_url(self, response, list_url_item): page_size = 10 use_url = list_url_item['use_url'] if response: use_url = response.url # old channel: no_parm_url = str(utils.get_url_no_query_params(use_url)) old_channel = no_parm_url.split('/')[-1] old_parsed_url = urlparse(use_url) old_query_params = old_parsed_url.query old_params = parse_qs(old_query_params) old_offset = int(old_params['offset'][0]) new_offset = old_offset + page_size list_url_item['use_url'] = self.get_list_url(old_channel, new_offset) # print(f'--- get_next_list_url result: {list_url_item["use_url"]}') return list_url_item def build_detail_urls(self, response:Response): print(f'--- response.headers:\n{response.headers}') detail_url_list = [] response_json = json.loads(response.text) if response_json and len(response_json) > 0: stories = response_json['stories'] for story in stories: detail_url = story['permalink'] detail_url_list.append(detail_url) # title = story['headline'] # print(f'\t--- detail_url: {detail_url}') return detail_url_list title_rules = [ { 'op': 'find', 'tag': 'div', 'attrs': [ { 'key': 'class_', 'type': 'eval', 'value': 're.compile("Box-")' } ] }, { 'op': 'find', 'tag': 'h1', 'attrs': [] }, { 'op': 'text' } ] # section main-column # content_rules = [ # { # 'op': 'find', # 'tag': 'section', # 'attrs': [ # { # 'key': 'id', # 'type': 'eval', # 'value': 're.compile("main-column")' # } # ] # }, # { # 'op': 'get_text' # } # ] from bs4 import BeautifulSoup def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('div', {'id': 'ad-in-text-target'}) if content_div: p_list = content_div.find_all('p') if len(p_list) > 0: for p in p_list: content_p = p.text.strip().replace('\n', '').replace('\t', '') content = content + '\n\n' + content_p except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ] img_href_rules = [ { 'op': 'find', 'tag': 'section', 'attrs': [ { 'key': 'id', 'type': 'eval', 'value': 're.compile("main-column")' } ] }, { 'op': 'find_all', 'tag': 'img', 'rules': [ { 'op': 'get_attr', 'name': 'src', 'no_query_params': True, 'with_origin_url': True } ] } ] author_rules = [ { 'op': 'find', 'tag': 'div', 'attrs': [ { 'key': 'class_', 'type': 'eval', 'value': 're.compile("Box-")' } ] }, { 'op': 'find', 'tag': 'a', 'attrs': [ { 'key': 'class_', 'type': 'eval', 'value': 're.compile("AuthorAnchor")' } ] }, { 'op': 'text' } ] time_rules = [ { 'op': 'find', 'tag': 'div', 'attrs': [ { 'key': 'class_', 'type': 'eval', 'value': 're.compile("Box-")' } ] }, { 'op': 'find', 'tag': 'span', 'attrs': [ { 'key': 'class_', 'type': 'eval', 'value': 're.compile("DateSpan")' } ] }, { 'op': 'text' } ]