", article_text)
title = str(re.sub(r'<[^>]+>', '', title_match.group(1)).strip()) if title_match else None
# Extract issue
issue_match = re.search(r'No\. (\d+)', article_text)
issue = issue_match.group(1) if issue_match else None
# Extract volume
volume_match = re.search(r'Vol\. (\d+)', article_text)
volume = str(volume_match.group(1)) if volume_match else None
if not volume:
volume_match = re.search(r'Special Issue (\d+) \((\d+)\)', article_text)
if volume_match:
issue_number, volume = volume_match.groups()
volume = str(volume)
issue = "Special Issue " + str(issue_number)
else:
volume_match = re.search(r'Conf\. (\d{2}), (\d{4})', article_text)
if volume_match:
issue = "Conference " + str(volume_match.group(1))
volume = str(volume_match.group(2))
else:
volume_match = re.search(r'Conference (\d+) \((\d+)\)', article_text)
if volume_match:
issue_number, volume = volume_match.groups()
volume = str(volume)
issue = "Conference " + str(issue_number)
else:
volume = None
# Extract pp
pp_match = re.search(r'pp\. (\d+-\d+)', article_text)
pp = pp_match.group(1) if pp_match else None
# Extract submission date
match = re.search(r"Submitted ([A-Za-z]+\s+\d{1,2}, \d{4})", html)
submitted_date = match.group(1) if match else None
if submitted_date:
submitted_date = datetime_transform(submitted_date)
# Extract publication date
match = re.search(r"Published ([A-Za-z]+\s+\d{1,2}, \d{4})", html)
publish_date = match.group(1) if match else None
if publish_date:
publish_date = datetime_transform(publish_date)
# Extract MSC
msc_match = re.search(r'Math Subject Classifications: (.*?)\n', html)
if not msc_match:
msc_match = re.search(r'Math Subject Classifications:(.*?)\.', html)
if msc_match:
msc = re.sub(r'<[^>]+>', '', msc_match.group(1).strip())
msc = msc.strip('.').strip()
msc = re.split(r', |;', msc)
else:
msc = []
# Extract KeyWords
keywords_match = re.search(r'Key Words: (.*?)(?:
|
|$)', html, re.DOTALL) if not keywords_match: keywords_match = re.search(r'Key Words: (.*?)(?=<)', html, re.DOTALL) if keywords_match: keywords = re.sub(r'<[^>]+>', '', keywords_match.group(1).strip()).replace('\n', '') keywords = re.split(r', |;', keywords) keywords = [unidecode(re.sub(r'\s+', ' ', keyword.strip().strip('.')).strip()) for keyword in keywords if len(keyword.strip())] else: keywords = [] # Extract DOI doi_match = re.search(r'DOI: ([^\t\n<]+)', html) if not doi_match: doi_match = re.search(r'DOI: (.+)', html) doi = doi_match.group(1) if doi_match else None doi = doi.replace('https://doi.org/', '') # strip doi website header # Article_id article_id = str(uuid.uuid4()) # Author info authors = [] table = article_soup.find('table') if table: for row in table.find_all('tr'): cells = [cell.text.strip() for cell in row.find_all('td')] for cell in cells: if "email:" in cell: cell = cell.split("email:") email_list = str(cell[1]).split(',') cell = cell[0] elif "e-mail:" in cell: cell = cell.split("e-mail:") email_list = str(cell[1]).split(',') cell = cell[0] else: email_list = None cell = re.split(r'[\r\n]+', cell) cell = [c.replace('\\newline', '') for c in cell] cell = [re.sub(r'\s+', ' ', c).strip() for c in cell] # Data processing if cell[0]: authors.append(unidecode(cell[0])) name = re.split(r'[ .]', cell[0]) affiliation = ', '.join(cell[1:]).lstrip(",").rstrip(",").strip() affiliation = re.sub(r'^[^a-zA-Z]*', '', re.sub(r'\s+', ' ', re.sub(r',+', ',', affiliation))) affiliation = affiliation.lstrip(",").rstrip(",").strip() emails = [] if email_list: for email in email_list: email_match = re.search(r'[\w.-]+@[\w.-]+', email) emails.append(unidecode(email_match.group())) if email_match else None author_data = { "author_id": str(uuid.uuid4()), "from_article": [article_id], "first_name": unidecode(name[0]), "last_name": unidecode(name[-1]), "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else None, "affiliation": [{ "year": volume, "affiliation": unidecode(affiliation), "email": emails }] } authorData.append(author_data) authorNum += 1 # If no author table else: match_type = 0 hr_count = len(article_soup.find_all('hr')) if hr_count < 3: pattern = r'
(.*?)
") for match in matches: if "email:" in match: match = match.split("email:") email_list = str(match[1]).split(',') match = match[0] elif "e-mail:" in match: match = match.split("e-mail:") email_list = str(match[1]).split(',') match = match[0] else: email_list = None match = re.sub(r'<[^>]+>', '', match) match = match.lstrip("\\n ").lstrip("\n ").rstrip("\\n ").rstrip("\n ").strip() if match_type == 0: match = match.split("\\n") else: match = match.split("\n") match = [m.replace('\\newline', '') for m in match] match = [re.sub(r'\s+', ' ', m).strip() for m in match] # Data processing if match[0]: authors.append(unidecode(match[0])) name = re.split(r'[ .]', match[0]) affiliation = ''.join(match[1:]).lstrip(",").rstrip(",").strip() affiliation = re.sub(r'^[^a-zA-Z]*', '', re.sub(r'\s+', ' ', re.sub(r',+', ',', affiliation))) affiliation = affiliation.lstrip(",").rstrip(",").strip() emails = [] if email_list: for email in email_list: email_match = re.search(r'[\w.-]+@[\w.-]+', email) emails.append(unidecode(email_match.group())) if email_match else None author_data = { "author_id": str(uuid.uuid4()), "from_article": [article_id], "first_name": unidecode(name[0]), "last_name": unidecode(name[-1]), "middle_name": unidecode(''.join(name[1:-1])) if len(name[1:-1]) > 0 else None, "affiliation": [{ "year": volume, "affiliation": unidecode(affiliation), "email": emails }] } authorData.append(author_data) authorNum += 1 else: print("AUTHOR SEARCHING ERROR:", article_url) fail = { "title": title, "URL": article_url } failedFormatData.append(fail) # Article info article_data = { "article_id": article_id, "title": unidecode(title), "authors": authors, "corresponding_authors": None, "submit_datetime": submitted_date, "publish_datetime": publish_date, "keywords": keywords, "MSC": msc, "URL": article_url, "DOI": doi, "publisher": "Texas State University", "journal": "Electronic Journal of Differential Equations", "volume": volume, "issue": issue, "page": pp } articleData.append(article_data) articleNum += 1 # Save the data periodically based on batch size if len(articleData) % batch_size == 0: ejde_save.save_data(articleData, "Article_TS", str(uuid.uuid4()) + ".json") articleData.clear() if len(authorData) % batch_size == 0: ejde_save.save_data(authorData, "Author_TS", str(uuid.uuid4()) + ".json") authorData.clear() start_time = time.time() url_list = [] # Get all general volumes url index = "https://ejde.math.txstate.edu/indexleft.html" response = requests.get(index) soup = BeautifulSoup(response.content, 'html.parser') volume_links = soup.select('font > a[href]') url_list.extend(["https://ejde.math.txstate.edu/" + link['href'] for link in volume_links[1:]][::-1]) # Get all special issues url index = "https://ejde.math.txstate.edu/special-toc.html" response = requests.get(index) soup = BeautifulSoup(response.content, 'html.parser') special_links = soup.find_all("a", href=True) url_list.extend(["https://ejde.math.txstate.edu/" + tag["href"] for tag in special_links[:-1]]) # Get all conference special issues url index = "https://ejde.math.txstate.edu/conf-toc.html#latest" response = requests.get(index) soup = BeautifulSoup(response.content, 'html.parser') special_links = soup.find_all("a", href=True) url_list.extend(["https://ejde.math.txstate.edu/" + tag["href"] for tag in special_links[:-2]]) authorData = [] articleData = [] failedData = [] totallyFailedData = [] failedVolData = [] failedFormatData = [] authorNum = 0 articleNum = 0 batch_size = 100 # Number of articles to process before saving executor = ThreadPoolExecutor(max_workers=int(len(url_list) / 2)) # Set the number of worker threads # Process each URL using multithreading futures = [executor.submit(process_volume, url) for url in url_list] # Wait for all tasks to complete for future in as_completed(futures): try: future.result() except Exception as vol_err: print("VOLUME PROCESSING ERROR:", str(vol_err)) # Retry failed processing paper print("START RETRYING:", len(failedData)) while failedData: data = failedData.pop(0) articleTitle = data["title"] articleUrl = data["URL"] try: process_article(articleTitle, articleUrl) except Exception as retry_err: print("ARTICLE RETRYING FAILURE:", str(retry_err)) totally_fail = { "title": articleTitle, "URL": articleUrl } totallyFailedData.append(totally_fail) # Save remaining data if len(articleData) > 0: ejde_save.save_data(articleData, "Article_TS", str(uuid.uuid4()) + ".json") print("COMPLETE: All EJDE article data has been saved to ./ejde_buffer/Article_TS/") if len(authorData) > 0: ejde_save.save_data(authorData, "Author_TS", str(uuid.uuid4()) + ".json") print("COMPLETE: All EJDE author data has been saved to ./ejde_buffer/Author_TS/") # Save error record if len(totallyFailedData) > 0: ejde_save.save_data(totallyFailedData, "Error", "Failed_article_record.json") print("Total failed processing paper:", len(totallyFailedData)) if len(failedVolData) > 0: ejde_save.save_data(failedVolData, "Error", "Failed_volume_record.json") print("Total failed fetching volume:", len(failedVolData)) if len(failedFormatData) > 0: ejde_save.save_data(failedFormatData, "Error", "Failed_format_record.json") print("Total failed searching article:", len(failedFormatData)) # Statistics print("Total fetched paper:", articleNum) print("Total fetched author:", authorNum) print("time elapsed: {:.2f}s".format(time.time() - start_time)) # Transfer to large file and delete the temporary storage files ejde_save.Transf() ejde_save.delete()