ldy/ScholarDataMining
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add code for saving data

Browse Source
This commit is contained in:
XCX 2023-08-09 12:22:42 +08:00
parent 9ee9bc4462
commit a9c753567c
1 changed files with 19 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
00_File_merge/Merge.py
View File

@ -1,5 +1,7 @@
import json import json
import os import os
from pprint import pprint
import unicodedata import unicodedata
''' '''
@ -63,6 +65,8 @@ def SameWeb_merge(folder_path):
# Update the counter # Update the counter
count[0] += 1 count[0] += 1
if count[0] % 100 == 0 and count[0] != 0:
print(str(count[0]) + ' copies of data have been merged.')
return Merge(a, count, Data) return Merge(a, count, Data)
# ========== Main code ========== # ========== Main code ==========
@ -86,13 +90,26 @@ def SameWeb_merge(folder_path):
Merge(a, count, Data) Merge(a, count, Data)
# Information # Information
print(str(count[0]) + ' copies of data have been merged.') print('\n========== Complete ==========\n')
print(str(Database) + ' copies of data in total, before') print(str(Database) + ' copies of data in total, before')
print(str(count[0]) + ' copies of data have been merged.')
print(str(len(Data)) + ' copies of data in total, now.') print(str(len(Data)) + ' copies of data in total, now.')
# Save into file
path = os.path.dirname(folder_path)
path = os.path.join(path, "Author_data(merged)")
os.makedirs(path, exist_ok=True)
path = os.path.join(path, "Author_data(merged).json")
with open(path, 'w', encoding='utf-8') as file:
json.dump(Data, file, indent=4)
print('\nData has been added to ' + path + '\Author_data(merged).json')
# =========== input the file path here ========== # =========== input the file path here ==========
# SameWeb_merge('./EJQTDE_buffer/Author_output') # SameWeb_merge('.\EJQTDE\EJQTDE_buffer\Author_output')
# SameWeb_merge('.\SpringerOpen\SpringerOpen_buffer\Author_output')