ldy/ScholarDataMining
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimization:

Browse Source 
clustered error files into a folder
This commit is contained in:
ldy 2023-08-11 11:42:02 +08:00
parent 7726650eaa
commit 35f5f2ac5e
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
01_EJDE_spider/ejde_main.py
View File

@ -155,6 +155,8 @@ def process_article(title, article_url):
# Extract volume # Extract volume
volume_match = re.search(r'Vol\. (\d+) \((\d+)\)', article_text) volume_match = re.search(r'Vol\. (\d+) \((\d+)\)', article_text)
volume = str(volume_match.group(1)) if volume_match else None volume = str(volume_match.group(1)) if volume_match else None
if not volume:
volume = str(volume_match.group(2)) if volume_match else None
# Extract pp # Extract pp
pp_match = re.search(r'pp\. (\d+-\d+)', article_text) pp_match = re.search(r'pp\. (\d+-\d+)', article_text)
@ -374,15 +376,15 @@ if len(authorData) > 0:
# Save error record # Save error record
if len(totallyFailedData) > 0: if len(totallyFailedData) > 0:
ejde_save.save_data(failedData, "", "Failed_article_record.json") ejde_save.save_data(failedData, "Error", "Failed_article_record.json")
print("Total failed processing paper:", len(totallyFailedData)) print("Total failed processing paper:", len(totallyFailedData))
if len(failedVolData) > 0: if len(failedVolData) > 0:
ejde_save.save_data(failedVolData, "", "Failed_volume_record.json") ejde_save.save_data(failedVolData, "Error", "Failed_volume_record.json")
print("Total failed fetching volume:", len(failedVolData)) print("Total failed fetching volume:", len(failedVolData))
if len(failedFormatData) > 0: if len(failedFormatData) > 0:
ejde_save.save_data(failedFormatData, "", "Failed_format_record.json") ejde_save.save_data(failedFormatData, "Error", "Failed_format_record.json")
print("Total failed searching article:", len(failedFormatData)) print("Total failed searching article:", len(failedFormatData))
# Total running time # Total running time