编辑采集器       终止采集器

属性	值	属性	值
名称	ph_inquirer	创建时间	2025-04-02 10:10:02
标题	菲律宾-Inquirer---(US-1-采)	更新时间	-
状态	running	开始时间	2025-07-04 00:26:12
停止类型		结束时间	-
信号		运行时长	89.8小时
进程ID	3839	进程名称	Process-7
作业ID	a9ea2d45-06c1-4ecf-a71c-a87cc86cd2af	作业目录	records/jobs/a9ea2d45-06c1-4ecf-a71c-a87cc86cd2af
日志文件	records/logs/a9ea2d45-06c1-4ecf-a71c-a87cc86cd2af.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	ph_inquirer_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider from scrapy.http import Response from bs4 import BeautifulSoup import utils import spider.spiders.common.rule as rule import logging logger = logging.getLogger(__name__) # ph_inquirer_spider.py class PhMindanaoDailyNewsSpider(BaseQueueListToDetailsSpider): name = 'ph_inquirer' desc = '菲律宾-Inquirer---(US-1-采)' net_name = 'Inquirer,Inquirer,菲律宾' net_url = 'www.inquirer.net' language = 'en' nation_category = 1 channel_name = 'News' channel_url = 'https://newsinfo.inquirer.net/category/latest-stories' channel_desc = 'Inquirer>News' max_page = 1000 now_page = 0 from_reached_url = False host_key = 'inquirer.net' exclude_urls = [] start_urls = [ { 'url':'https://newsinfo.inquirer.net/category/latest-stories', 'max_page': 2, 'language': 'en', 'nation_category': 1, 'channel_name': 'News', 'channel_url': 'https://newsinfo.inquirer.net/category/latest-stories', 'channel_desc': 'Inquirer>News' }, ] # https://newsinfo.inquirer.net/category/latest-stories # https://newsinfo.inquirer.net/category/latest-stories/page/2 def build_next_list_url(self, response, list_url_item): use_url = list_url_item['use_url'] if response: current_url = str(response.url) # current_url = utils.cut_last_str(current_url, '/') page = 1 url_file_name = str(utils.get_url_file_name(current_url)) if url_file_name.isdigit(): page = int(url_file_name) + 1 response_url_no_file = utils.cut_last_str(current_url, url_file_name) use_url = f'{response_url_no_file}{page}' else: page = page + 1 use_url = f'{current_url}/page/{page}' list_url_item['use_url'] = use_url # print(f'--- get_next_list_url final: {use_url}') return list_url_item list_hrefs_rules = [ {'op': 'find_tag_by_id', 'tag': 'section', 'id': 'inq_section'}, {'op': 'find_div_by_id', 'id': 'inq-channel-left'}, { 'op': 'find_all', 'tag': 'h2', 'type_str_filter': 'Tag', # 类型过滤器，根据Tag名称的字符串 'attrs': [], 'rules': [ {'op': 'find','tag': 'a'}, {'op': 'get_href'} ] } ] title_rules = [ {'op': 'find_div_by_id', 'id': 'art-head-group'}, {'op': 'find', 'tag': 'h1', 'attrs': []}, {'op': 'text', 'eval': 'text.strip()'} ] author_rules = [ {'op': 'find_div_by_id', 'id': 'art-head-group'}, {'op': 'find_div_by_id', 'id': 'art_author'}, {'op': 'find', 'tag': 'a', 'attrs': []}, {'op': 'text', 'eval': 'text.strip()'} ] time_rules = [ {'op': 'find_div_by_id', 'id': 'art-head-group'}, {'op': 'find_div_by_id', 'id': 'art_plat'}, {'op': 'text', 'eval': 'text.strip().split("/")[1].strip() if len(text.strip().split("/"))>1 else text'} ] # content_rules = [ # {'op': 'find_div_by_id', 'id': 'art_body_wrap'}, # {'op': 'find_div_by_id', 'id': 'article_content'}, # {'op': 'text', 'eval': 'text.strip().split("Your subscription")[0].strip() if len(text.strip().split("Your subscription"))>0 else text'} # ] #mgid_target_content from bs4 import BeautifulSoup def get_content(soup: BeautifulSoup): content = '' try: content_div = soup.find('div', {'id': 'mgid_target_content'}) if content_div: p_list = content_div.find_all('p') if len(p_list) > 0: for p in p_list: content_p = p.text.strip().replace('\n', '').replace('\t', '') content = content + '\n\n' + content_p except Exception as e: utils.log_error(f'get img fail: {e}') return content content_rules = [ { 'op': 'func', 'name': get_content } ] img_href_ignore_starts = ['data:'] img_href_rules = [ {'op': 'find_div_by_id', 'id': 'art_body_wrap'}, {'op': 'find_div_by_id', 'id': 'article_content'}, { 'op': 'find_all', 'tag': 'img', 'rules': [ { 'op': 'get_attr', 'name': 'src' } ] } ]