175 lines
4.3 KiB
Plaintext
175 lines
4.3 KiB
Plaintext
import requests
|
||
from bs4 import BeautifulSoup
|
||
import re
|
||
import uuid
|
||
from bs4 import Tag
|
||
import json
|
||
|
||
|
||
# 发送HTTP请求获取网页内容
|
||
url = 'https://www.worldscientific.com/doi/10.1142/S0219891623500017'
|
||
response = requests.get(url)
|
||
html_content = response.content
|
||
|
||
# 使用BeautifulSoup解析网页内容
|
||
soup = BeautifulSoup(html_content, 'html.parser')
|
||
|
||
# 提取标题
|
||
title = soup.find(class_="citation__title").text
|
||
# 打印标题
|
||
print(title)
|
||
|
||
|
||
# 提取所有class为"author-type"的内容
|
||
author_type_tags = soup.find_all(class_="author-type")
|
||
|
||
# 获取提取到的内容
|
||
author_type_list = [tag.text.strip() for tag in author_type_tags]
|
||
|
||
# 输出提取的内容
|
||
print(author_type_list)
|
||
|
||
for author_type_tag in author_type_tags:
|
||
# 提取后续的 <p> 标签内容
|
||
affiliation = author_type_tag.find_next_sibling('p')
|
||
if affiliation:
|
||
content_affiliation = affiliation.get_text()
|
||
print(content_affiliation)
|
||
else:
|
||
print("No <p> tag found after class=\"author-type\"")
|
||
|
||
|
||
|
||
|
||
|
||
# 提取作者姓名，创建作者字典
|
||
author_tags = soup.find(class_="rlist--inline loa mobile-authors visible-xs").text
|
||
|
||
# 去除额外空格
|
||
author_tags = author_tags.replace(" ", "")
|
||
|
||
# 使用逗号拆分作者标签
|
||
author_list = author_tags.split("and")
|
||
|
||
authors_dict = {}
|
||
for i, author in enumerate(author_list, 1):
|
||
# 去除多余空格
|
||
author = author.strip()
|
||
authors_dict[f"author_{i}"] = author
|
||
authors_dict[f"affiliation_{i}"]=affiliation
|
||
|
||
print(authors_dict)
|
||
|
||
# 提取关键词
|
||
# 找到包含keywords的div元素
|
||
keywords_div = soup.find('div', id='keywords')
|
||
|
||
# 提取keywords内容
|
||
keywords = []
|
||
if keywords_div:
|
||
keyword_items = keywords_div.find_all('a')
|
||
keywords = [item.text for item in keyword_items]
|
||
|
||
print(keywords)
|
||
|
||
|
||
|
||
# 定义匹配要提取的div的正则表达式
|
||
pattern = re.compile(r'^<div><b>AMSC: </b>.*</div>$')
|
||
|
||
# 找到匹配正则表达式的div元素
|
||
divs = soup.find_all(lambda tag: tag.name == 'div' and re.match(pattern, str(tag)))
|
||
|
||
# 提取每个div中的AMSC内容
|
||
for div in divs:
|
||
amsc_text = div.text
|
||
# 提取AMSC值（去除"AMSC: "前缀）
|
||
amsc_values = amsc_text.replace('AMSC: ', '').strip().split(', ')
|
||
|
||
print("AMSC", amsc_values)
|
||
|
||
|
||
# 找到提交时间，定义匹配模式
|
||
pattern = re.compile('^received', re.IGNORECASE)
|
||
|
||
# 查找匹配的div元素
|
||
received_div = soup.find('div', string=pattern)
|
||
|
||
# 提取内容
|
||
if received_div:
|
||
received_text = received_div.text
|
||
|
||
print(received_text)
|
||
|
||
|
||
|
||
# 找到发表时间，定义匹配模式
|
||
pattern1 = re.compile('^Published', re.IGNORECASE)
|
||
|
||
# 查找匹配的div元素
|
||
Published_div = soup.find('div', string=pattern1)
|
||
|
||
# 提取内容
|
||
if Published_div:
|
||
Published_text = Published_div.text
|
||
|
||
print(Published_text)
|
||
|
||
# 查找DOI标签并提取DOI
|
||
doi_element = soup.find("span", class_="epub-section__item").find("a")
|
||
doi = doi_element['href'].split("doi.org")[-1]
|
||
|
||
print("DOI:", doi)
|
||
|
||
|
||
|
||
# 查找具有class为article__tocHeading的链接
|
||
links = soup.find_all('a', class_='article__tocHeading')
|
||
|
||
# 遍历找到的链接
|
||
for link in links:
|
||
text = link.text # 链接文本内容
|
||
|
||
# 使用正则表达式提取数据
|
||
match = re.search(r'Vol\. (\d+), No\. (\d+), pp\. (\d+-\d+) \((\d+)\)', text)
|
||
|
||
if match:
|
||
vol_number = match.group(1)
|
||
issue_number = match.group(2)
|
||
page_range = match.group(3)
|
||
publication_year = match.group(4)
|
||
|
||
# 打印提取的数据
|
||
print("Volume Number:", vol_number)
|
||
print("Issue Number:", issue_number)
|
||
print("Page Range:", page_range)
|
||
print("Publication Year:", publication_year)
|
||
else:
|
||
print("No match found.")
|
||
|
||
data={
|
||
"article_id":str(uuid.uuid4()),
|
||
"Author":authors_dict,
|
||
"Title":title,
|
||
"received time":received_text,
|
||
"publish time":Published_text,
|
||
"keyeords":keywords,
|
||
"Volume":vol_number,
|
||
"issue":issue_number,
|
||
"page":page_range,
|
||
"DOI":doi,
|
||
"url":"",
|
||
"journal":"worldscientific",
|
||
"MSC":amsc_values}
|
||
|
||
|
||
class CustomJSONEncoder(json.JSONEncoder):
|
||
def default(self, obj):
|
||
if isinstance(obj, Tag):
|
||
return str(obj)
|
||
return super().default(obj)
|
||
|
||
# 使用自定义JSON编码器进行序列化
|
||
with open("data.json", "w", encoding="utf-8") as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False, cls=CustomJSONEncoder)
|