编辑采集器       终止采集器

属性	值	属性	值
名称	ph_coconuts_news_history	创建时间	2025-04-02 10:10:02
标题	菲律宾-Coconuts news 已停止更新-历史数据---(US-2-采)（zxc）	更新时间	-
状态	running	开始时间	2025-07-04 00:26:19
停止类型		结束时间	-
信号		运行时长	90.6小时
进程ID	3848	进程名称	Process-10
作业ID	23bba64c-5ad1-40d9-a5a1-94e0801628b3	作业目录	records/jobs/23bba64c-5ad1-40d9-a5a1-94e0801628b3
日志文件	records/logs/23bba64c-5ad1-40d9-a5a1-94e0801628b3.log     (查看日志)     (tail日志)
采集器类型	scrapy	文件	ph_coconuts_news_history_spider.py
起始urls(每行一个) (若为空则使用代码中 的start_urls)

+新增命令

启动命令	名称	类型	命令内容	状态	创建时间	启动时间	结束时间	时长	修改	删除

+新增定时任务

定时任务	类型	定时时间	更新时间	激活	修改	删除

历史作业 ID	开始时间	结束时间	运行时长	进程ID	进程名	日志
删除最后 选择条数 条历史作业 确定

代码

import bs4 import logging import json from scrapy.http import Response from urllib.parse import urlparse, parse_qs, urlencode from spider.spiders.common.base_spider import BaseQueueListToDetailsSpider logger = logging.getLogger(__name__) class PhCoconutsNewsHistorySpider(BaseQueueListToDetailsSpider): name = "ph_coconuts_news_history" desc = '菲律宾-Coconuts news 已停止更新-历史数据---(US-2-采)（zxc）' net_name = 'Coconuts,Coconuts新闻,菲律宾' net_url = 'https://coconuts.co/manila/news/' language = 'en' nation_category = 1 channel_name = '新闻' channel_url = 'https://coconuts.co/manila/news/' channel_desc = 'coconuts>manila' max_page = 50000 now_page = 0 from_reached_url = False host_key = 'coconuts.co' exclude_urls = [] start_urls = [ { 'url': 'https://coconuts.co/wp-admin/admin-ajax.php?action=load_more_ajax&paged=1&category=600018', 'max_page': 10000, 'language': 'en', 'nation_category': 1, 'channel_name': 'manila', 'channel_url': 'https://coconuts.co/manila/news/', 'channel_desc': 'coconuts>manila' }] def build_next_list_url(self, response, list_url_item): if response: #resp_json = json.loads(response.text) # next_link = resp_json['nextLink'] # print('next_link') # print(next_link) url = response.url parsed_url = urlparse(url) query_params = parse_qs(parsed_url.query) current_page = int(query_params.get('paged', [1])[0]) new_page = current_page + 1 query_params['paged'] = [f'{new_page}'] # Rebuild the URL new_query = urlencode(query_params, doseq=True) new_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}?{new_query}" print(new_url) list_url_item['use_url'] = new_url return list_url_item #https://coconuts.co/manila/news/c ebu-pacific-vows-to-address-flight-disruptions-passenger-woes-after-lawmaker-proposes-suspension def build_detail_urls(self, response: Response): detail_url_list = [] response_json = json.loads(response.text) if response_json and response_json['results'] : data_html = response_json['results'] clean_html_content = data_html.replace('\n', '').replace('\t', '') soup = bs4.BeautifulSoup(clean_html_content, 'html.parser') div_a = soup.find('div', class_='col-6 col-lg-4 h-archive') a_list = div_a.find_all('a', href=True) for a in a_list: detail_url_list.append(a.get('href')) return detail_url_list title_rules = [ {'op': 'find_tag_by_class', 'tag': 'h1', 'class_': 'post-title'}, {'op': 'text'} ] author_rules = [ {'op': 'find_tag_by_class', 'tag': 'div', 'class_': 'd-flex align-items-center'}, {'op': 'find','tag': 'span'}, {'op': 'find','tag': 'a'}, {'op': 'text'} ] time_rules = [ {'op': 'find_tag_by_class', 'tag': 'div', 'class_': 'post-timeago'}, {'op': 'text','remove':'|'} ] def get_content(soup: bs4.BeautifulSoup): con_div = soup.find('div', class_='coco_post-content') con_ps = con_div.find_all('p') content = '' for p in con_ps: if p.text: paragraph = p.text.strip() content = content + '\n\n' + paragraph return content.strip() content_rules = [{ 'op': 'func', 'name': get_content } ] img_href_rules = [ {'op': 'find_tag_by_class', 'tag': 'figure', 'class_': 'post-image'}, {'op': 'find_all','tag': 'img','rules': [ { 'op': 'get_attr', 'name': 'src' } ] } ]