97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
import os
|
|
import json
|
|
import uuid
|
|
|
|
|
|
# Save into files
|
|
def save_data(dataset, filetype):
|
|
if dataset:
|
|
filename = str(uuid.uuid4()) + ".json"
|
|
directory = "./EJQTDE_buffer/" + filetype + "/"
|
|
os.makedirs(directory, exist_ok=True)
|
|
filepath = os.path.join(directory, filename)
|
|
with open(filepath, "w", encoding='utf-8') as json_file:
|
|
json.dump(dataset, json_file, indent=4)
|
|
print(filetype + " data have been added to", filepath)
|
|
|
|
|
|
# Summary files
|
|
def Transf():
|
|
def Read(folder_path, output_files):
|
|
# Create new folder
|
|
os.makedirs('./EJQTDE_buffer/Article_output/', exist_ok=True)
|
|
os.makedirs('./EJQTDE_buffer/Author_output/', exist_ok=True)
|
|
|
|
data_oldest = []
|
|
data_2010_2014 = []
|
|
data_2015_2020 = []
|
|
data_newest = []
|
|
|
|
for filename in os.listdir(folder_path):
|
|
if filename.endswith('.json'):
|
|
file_path = os.path.join(folder_path, filename)
|
|
with open(file_path, 'r', encoding='utf-8') as file:
|
|
data = json.load(file)
|
|
|
|
for Dict in data:
|
|
if Dict.get('volume') and Dict.get('affiliation', [{}])[0].get('year', 0) is not None:
|
|
# Select data
|
|
data_oldest += [Dict for Dict in data if (isinstance(Dict, dict) and int(
|
|
Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009)]
|
|
|
|
data_2010_2014 += [Dict for Dict in data if (isinstance(Dict, dict) and 2010 <= int(
|
|
Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014)]
|
|
|
|
data_2015_2020 += [Dict for Dict in data if (isinstance(Dict, dict) and 2015 <= int(
|
|
Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020)]
|
|
|
|
data_newest += [Dict for Dict in data if (isinstance(Dict, dict) and int(
|
|
Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) >= 2021)]
|
|
|
|
Data = [data_oldest, data_2010_2014, data_2015_2020, data_newest]
|
|
|
|
# Transfer
|
|
for index in range(0, 4):
|
|
with open(output_files[index], 'w', encoding='utf-8') as file:
|
|
json.dump(Data[index], file, indent=4)
|
|
|
|
# The path of reading
|
|
author_folder_path = './EJQTDE_buffer/Author_TS'
|
|
article_folder_path = './EJQTDE_buffer/Article_TS'
|
|
|
|
# The path of storage
|
|
author_output_file = [
|
|
'./EJQTDE_buffer/Author_output/Author_output_file(oldest).json',
|
|
'./EJQTDE_buffer/Author_output/Author_output_file(2010-2014).json',
|
|
'./EJQTDE_buffer/Author_output/Author_output_file(2015-2020).json',
|
|
'./EJQTDE_buffer/Author_output/Author_output_file(newest).json'
|
|
]
|
|
|
|
article_output_file = [
|
|
'./EJQTDE_buffer/Article_output/Article_output_file(oldest).json',
|
|
'./EJQTDE_buffer/Article_output/Article_output_file(2010-2014).json',
|
|
'./EJQTDE_buffer/Article_output/Article_output_file(2015-2020).json',
|
|
'./EJQTDE_buffer/Article_output/Article_output_file(newest).json'
|
|
]
|
|
|
|
# Read and write into files
|
|
Read(author_folder_path, author_output_file)
|
|
Read(article_folder_path, article_output_file)
|
|
|
|
# End
|
|
print("\nData has been written into files.")
|
|
|
|
|
|
# Delete files in temporary storage area
|
|
def delete():
|
|
folder_paths = ['./EJQTDE_buffer/Author_TS', './EJQTDE_buffer/Article_TS']
|
|
for folder_path in folder_paths:
|
|
file_names = os.listdir(folder_path)
|
|
for file_name in file_names:
|
|
file_path = os.path.join(folder_path, file_name)
|
|
if os.path.isfile(file_path):
|
|
os.remove(file_path)
|
|
|
|
print('\nAttention: The temporary storage files have been deleted!')
|
|
|