", article_text) - title = str(re.sub(r'<[^>]+>', '', title_match.group(1)).strip()) if title_match else None + title = str(re.sub(r'<[^>]+>', '', title_match.group(1)).strip()) if title_match else "" # Extract issue issue_match = re.search(r'No\. (\d+)', article_text) - issue = issue_match.group(1) if issue_match else None + issue = issue_match.group(1) if issue_match else "" # Extract volume volume_match = re.search(r'Vol\. (\d+)', article_text) @@ -189,21 +196,23 @@ def process_article(title, article_url): volume = str(volume) issue = "Conference " + str(issue_number) else: - volume = None + volume = "" # Extract pp pp_match = re.search(r'pp\. (\d+-\d+)', article_text) - pp = pp_match.group(1) if pp_match else None + pp = pp_match.group(1) if pp_match else "" # Extract submission date match = re.search(r"Submitted ([A-Za-z]+\s+\d{1,2}, \d{4})", html) - submitted_date = match.group(1) if match else None + if not match: + match = re.search(r"Submitted\s+([A-Za-z]+\s+\d{1,2},\s+\d{4})", html) + submitted_date = match.group(1) if match else "" if submitted_date: submitted_date = datetime_transform(submitted_date) # Extract publication date match = re.search(r"Published ([A-Za-z]+\s+\d{1,2}, \d{4})", html) - publish_date = match.group(1) if match else None + publish_date = match.group(1) if match else "" if publish_date: publish_date = datetime_transform(publish_date) @@ -234,25 +243,25 @@ def process_article(title, article_url): doi_match = re.search(r'DOI: ([^\t\n<]+)', html) if not doi_match: doi_match = re.search(r'DOI: (.+)', html) - doi = doi_match.group(1) if doi_match else None - doi = doi.replace('https://doi.org/', '') # strip doi website header + doi = doi_match.group(1) if doi_match else "" # Article_id article_id = str(uuid.uuid4()) # Author info authors = [] + author_names = [] table = article_soup.find('table') if table: for row in table.find_all('tr'): cells = [cell.text.strip() for cell in row.find_all('td')] for cell in cells: - if "email:" in cell: - cell = cell.split("email:") + if "email" in cell: + cell = cell.split("email") email_list = str(cell[1]).split(',') cell = cell[0] - elif "e-mail:" in cell: - cell = cell.split("e-mail:") + elif "e-mail" in cell: + cell = cell.split("e-mail") email_list = str(cell[1]).split(',') cell = cell[0] else: @@ -264,8 +273,11 @@ def process_article(title, article_url): # Data processing if cell[0]: - authors.append(unidecode(cell[0])) - name = re.split(r'[ .]', cell[0]) + author_id = str(uuid.uuid4()) + authors.append(author_id) + author_names.append(unidecode(cell[0])) + name = re.split(r'\s+', cell[0]) + name = [item for item in name if item != ''] affiliation = ', '.join(cell[1:]).lstrip(",").rstrip(",").strip() affiliation = re.sub(r'^[^a-zA-Z]*', '', re.sub(r'\s+', ' ', re.sub(r',+', ',', affiliation))) affiliation = affiliation.lstrip(",").rstrip(",").strip() @@ -276,16 +288,19 @@ def process_article(title, article_url): emails.append(unidecode(email_match.group())) if email_match else None author_data = { - "author_id": str(uuid.uuid4()), - "from_article": [article_id], + "author_id": author_id, + "from_article": article_id, "first_name": unidecode(name[0]), "last_name": unidecode(name[-1]), - "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else None, - "affiliation": [{ - "year": volume, - "affiliation": unidecode(affiliation), - "email": emails - }] + "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else "", + "raw_name": unidecode(cell[0]), + "affiliation": [ + { + "year": volume, + "affiliation": unidecode(affiliation), + "email": ", ".join(emails) + } + ] } authorData.append(author_data) authorNum += 1 @@ -308,12 +323,12 @@ def process_article(title, article_url): matches = matches.replace('["', '').replace('"]', '').replace('[\'', '').replace('\']', '') matches = matches.split("
") for match in matches: - if "email:" in match: - match = match.split("email:") + if "email" in match: + match = match.split("email") email_list = str(match[1]).split(',') match = match[0] - elif "e-mail:" in match: - match = match.split("e-mail:") + elif "e-mail" in match: + match = match.split("e-mail") email_list = str(match[1]).split(',') match = match[0] else: @@ -330,8 +345,11 @@ def process_article(title, article_url): # Data processing if match[0]: + author_id = str(uuid.uuid4()) + authors.append(author_id) authors.append(unidecode(match[0])) - name = re.split(r'[ .]', match[0]) + name = re.split(r'\s+', match[0]) + name = [item for item in name if item != ''] affiliation = ''.join(match[1:]).lstrip(",").rstrip(",").strip() affiliation = re.sub(r'^[^a-zA-Z]*', '', re.sub(r'\s+', ' ', re.sub(r',+', ',', affiliation))) affiliation = affiliation.lstrip(",").rstrip(",").strip() @@ -342,16 +360,19 @@ def process_article(title, article_url): emails.append(unidecode(email_match.group())) if email_match else None author_data = { - "author_id": str(uuid.uuid4()), - "from_article": [article_id], + "author_id": author_id, + "from_article": article_id, "first_name": unidecode(name[0]), "last_name": unidecode(name[-1]), - "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else None, - "affiliation": [{ - "year": volume, - "affiliation": unidecode(affiliation), - "email": emails - }] + "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else "", + "raw_name": unidecode(match[0]), + "affiliation": [ + { + "year": volume, + "affiliation": unidecode(affiliation), + "email": ", ".join(emails) + } + ] } authorData.append(author_data) authorNum += 1 @@ -368,7 +389,7 @@ def process_article(title, article_url): "article_id": article_id, "title": unidecode(title), "authors": authors, - "corresponding_authors": None, + "author_names": author_names, "submit_datetime": submitted_date, "publish_datetime": publish_date, "keywords": keywords, @@ -386,12 +407,10 @@ def process_article(title, article_url): # Save the data periodically based on batch size if len(articleData) % batch_size == 0: - ejde_save.save_data(articleData, "Article_TS", str(uuid.uuid4()) + ".json") - articleData.clear() + save_data_thread_safe(articleData, articleDataLock, "Article_TS") if len(authorData) % batch_size == 0: - ejde_save.save_data(authorData, "Author_TS", str(uuid.uuid4()) + ".json") - authorData.clear() + save_data_thread_safe(authorData, authorDataLock, "Author_TS") start_time = time.time() @@ -429,6 +448,8 @@ authorNum = 0 articleNum = 0 batch_size = 100 # Number of articles to process before saving +authorDataLock = threading.Lock() +articleDataLock = threading.Lock() executor = ThreadPoolExecutor(max_workers=int(len(url_list) / 2)) # Set the number of worker threads # Process each URL using multithreading @@ -444,9 +465,9 @@ for future in as_completed(futures): # Retry failed processing paper print("START RETRYING:", len(failedData)) while failedData: - data = failedData.pop(0) - articleTitle = data["title"] - articleUrl = data["URL"] + fail_data = failedData.pop(0) + articleTitle = fail_data["title"] + articleUrl = fail_data["URL"] try: process_article(articleTitle, articleUrl) except Exception as retry_err: @@ -460,11 +481,11 @@ while failedData: # Save remaining data if len(articleData) > 0: ejde_save.save_data(articleData, "Article_TS", str(uuid.uuid4()) + ".json") - print("COMPLETE: All EJDE article data has been saved to ./ejde_buffer/Article_TS/") +print("COMPLETE: All EJDE article data has been saved to ./ejde_buffer/Article_TS/") if len(authorData) > 0: ejde_save.save_data(authorData, "Author_TS", str(uuid.uuid4()) + ".json") - print("COMPLETE: All EJDE author data has been saved to ./ejde_buffer/Author_TS/") +print("COMPLETE: All EJDE author data has been saved to ./ejde_buffer/Author_TS/") # Save error record if len(totallyFailedData) > 0: @@ -485,5 +506,5 @@ print("Total fetched author:", authorNum) print("time elapsed: {:.2f}s".format(time.time() - start_time)) # Transfer to large file and delete the temporary storage files -ejde_save.Transf() -ejde_save.delete() +ejde_save.transform_data() +# ejde_save.delete_data() diff --git a/Parsers/01_EJDE_spider/ejde_save.py b/Parsers/01_EJDE_spider/ejde_save.py index 7ca56d9..1c6f95b 100644 --- a/Parsers/01_EJDE_spider/ejde_save.py +++ b/Parsers/01_EJDE_spider/ejde_save.py @@ -14,8 +14,8 @@ def save_data(dataset, filetype, filename): # Write into output files -def Transf(): - def Read(folder_path, output_files): +def transform_data(): + def read(folder_path, output_files): # Create new folders os.makedirs('./ejde_buffer/Article_output/', exist_ok=True) os.makedirs('./ejde_buffer/Author_output/', exist_ok=True) @@ -24,6 +24,8 @@ def Transf(): data_2010_2014 = [] data_2015_2020 = [] data_newest = [] + data_no_date = [] + data_integrate = [] for filename in os.listdir(folder_path): if filename.endswith('.json'): @@ -31,24 +33,29 @@ def Transf(): with open(file_path, 'r', encoding='utf-8') as file: data = json.load(file) for Dict in data: - if Dict.get('volume') is not None or Dict.get('affiliation', [{}])[0].get('year', 0) is not None: + if Dict.get('volume') is not None or Dict.get('affiliation', [{}])[0].get('year', 0) != '': # Select data if int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009: data_oldest.append(Dict) - elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014: data_2010_2014.append(Dict) - elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020: data_2015_2020.append(Dict) - else: data_newest.append(Dict) + else: + data_no_date.append(Dict) + + data_integrate.append(data_oldest) + data_integrate.append(data_2010_2014) + data_integrate.append(data_2015_2020) + data_integrate.append(data_newest) + data_integrate.append(data_no_date) # Transfer - Data = [data_oldest, data_2010_2014, data_2015_2020, data_newest] + Data = [data_oldest, data_2010_2014, data_2015_2020, data_newest, data_no_date, data_integrate] - for index in range(0, 4): + for index in range(0, 6): with open(output_files[index], 'w', encoding='utf-8') as file: json.dump(Data[index], file, indent=4) @@ -61,26 +68,30 @@ def Transf(): './ejde_buffer/Author_output/EJDE_Author_output_file(oldest).json', './ejde_buffer/Author_output/EJDE_Author_output_file(2010-2014).json', './ejde_buffer/Author_output/EJDE_Author_output_file(2015-2020).json', - './ejde_buffer/Author_output/EJDE_Author_output_file(newest).json' + './ejde_buffer/Author_output/EJDE_Author_output_file(newest).json', + './ejde_buffer/Author_output/EJDE_Author_output_file(no date).json', + './ejde_buffer/Author_output/EJDE_Author_output_file(integration).json' ] article_output_file = [ './ejde_buffer/Article_output/EJDE_Article_output_file(oldest).json', './ejde_buffer/Article_output/EJDE_Article_output_file(2010-2014).json', './ejde_buffer/Article_output/EJDE_Article_output_file(2015-2020).json', - './ejde_buffer/Article_output/EJDE_Article_output_file(newest).json' + './ejde_buffer/Article_output/EJDE_Article_output_file(newest).json', + './ejde_buffer/Article_output/EJDE_Article_output_file(no date).json', + './ejde_buffer/Article_output/EJDE_Article_output_file(integration).json' ] # Read and write into files - Read(author_folder_path, author_output_file) - Read(article_folder_path, article_output_file) + read(author_folder_path, author_output_file) + read(article_folder_path, article_output_file) # End print("\nData has been written into files.") # Delete files in temporary storage area -def delete(): +def delete_data(): folder_paths = ['./ejde_buffer/Author_TS', './ejde_buffer/Article_TS'] for folder_path in folder_paths: file_names = os.listdir(folder_path) @@ -89,5 +100,4 @@ def delete(): if os.path.isfile(file_path): os.remove(file_path) os.rmdir(folder_path) - print('\nAttention: The temporary storage files have been deleted!')