ldy/ScholarDataMining
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed the data structure

Browse Source
This commit is contained in:
XCX 2023-08-18 19:43:22 +08:00
parent 1d79556c42
commit 88bcbf5b8f
3 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
01_EJDE_spider/ejde_main.py
View File

@ -240,7 +240,7 @@ def process_article(title, article_url):
"from_article": [article_id], "from_article": [article_id],
"first_name": name[0], "first_name": name[0],
"last_name": name[-1], "last_name": name[-1],
"middle_name": ''.join(name[1:-1]) if name[1:-1] else None, "middle_name": ''.join(name[1:-1]) if len(name[1:-1]) > 0 else None,
"affiliation": [{ "affiliation": [{
"year": volume, "year": volume,
"affiliation": affiliation, "affiliation": affiliation,
@ -287,10 +287,10 @@ def process_article(title, article_url):
author_data = { author_data = {
"author_id": str(uuid.uuid4()), "author_id": str(uuid.uuid4()),
"from_article": article_id, "from_article": [article_id],
"first_name": name[0], "first_name": name[0],
"last_name": name[-1], "last_name": name[-1],
"middle_name": ''.join(name[1:-1]) if name[1:-1] else None, "middle_name": ''.join(name[1:-1]) if len(name[1:-1]) > 0 else None,
"affiliation": [{ "affiliation": [{
"year": volume, "year": volume,
"affiliation": affiliation, "affiliation": affiliation,

2
02_EJQTDE_spider/ejqtde_scrawler.py
View File

@ -48,7 +48,7 @@ def author_detail(Data, Year, article_id, Author_list):
Firstname = author[0] Firstname = author[0]
Lastname = author[-1] Lastname = author[-1]
Middlename = ''.join(author[1:-1]) if author[1:-1] else None Middlename = ''.join(author[1:-1]) if len(author[1:-1]) > 0 else None
# infor # infor
table = Data.find('table', attrs={'border': '1', 'cellpadding': '2px'}) table = Data.find('table', attrs={'border': '1', 'cellpadding': '2px'})

2
04_SpringerOpen_spider/SD_detail.py
View File

@ -20,7 +20,7 @@ def Author_dict(soup, article_id, Author_list):
Firstname = author[0] Firstname = author[0]
Lastname = author[-1] Lastname = author[-1]
Middlename = ''.join(author[1:-1]) if author[1:-1] else None Middlename = ''.join(author[1:-1]) if len(author[1:-1]) > 0 else None
# Year # Year
Year = info.find('span', attrs={'data-test': 'article-publication-year'}).get_text() if info.find\ Year = info.find('span', attrs={'data-test': 'article-publication-year'}).get_text() if info.find\