From ef9ab9abb16dbc5e17eb6532e3d9a5cc35f4e897 Mon Sep 17 00:00:00 2001 From: ldy Date: Tue, 26 Sep 2023 23:37:43 +0800 Subject: [PATCH] Optimized file allocation --- {00_Data => Data}/Merged/EJQTDE_buffer.zip | Bin {00_Data => Data}/Merged/ejde_buffer.zip | Bin {00_Data => Data}/Origin/EJQTDE_buffer.zip | Bin {00_Data => Data}/Origin/ejde_buffer.zip | Bin {00_File_merge => FileMerger}/Division_byName.py | 0 {00_File_merge => FileMerger}/Merge.py | 0 {00_File_merge => FileMerger}/fileReader.py | 0 .../00_Citation_spider}/get_paper_citation.py | 0 .../01_EJDE_spider}/ejde_main.py | 0 .../01_EJDE_spider}/ejde_save.py | 0 .../02_EJQTDE_spider}/ejqtde_main.py | 0 .../02_EJQTDE_spider}/ejqtde_save.py | 0 .../02_EJQTDE_spider}/ejqtde_scrawler.py | 0 .../03_ProjectEuclid_spider}/projecteuclid_main | 0 .../04_SpringerOpen_spider}/SD_detail.py | 0 .../04_SpringerOpen_spider}/SD_link.py | 0 .../04_SpringerOpen_spider}/SD_main.py | 0 .../04_SpringerOpen_spider}/SD_save.py | 0 .../04_SpringerOpen_spider}/SD_scrawl.py | 4 ++-- .../04_SpringerOpen_spider}/SD_threads.py | 0 .../05_WorldScientific_spider}/WorldScientific | 0 21 files changed, 2 insertions(+), 2 deletions(-) rename {00_Data => Data}/Merged/EJQTDE_buffer.zip (100%) rename {00_Data => Data}/Merged/ejde_buffer.zip (100%) rename {00_Data => Data}/Origin/EJQTDE_buffer.zip (100%) rename {00_Data => Data}/Origin/ejde_buffer.zip (100%) rename {00_File_merge => FileMerger}/Division_byName.py (100%) rename {00_File_merge => FileMerger}/Merge.py (100%) rename {00_File_merge => FileMerger}/fileReader.py (100%) rename {01_EJDE_spider => Parsers/00_Citation_spider}/get_paper_citation.py (100%) rename {01_EJDE_spider => Parsers/01_EJDE_spider}/ejde_main.py (100%) rename {01_EJDE_spider => Parsers/01_EJDE_spider}/ejde_save.py (100%) rename {02_EJQTDE_spider => Parsers/02_EJQTDE_spider}/ejqtde_main.py (100%) rename {02_EJQTDE_spider => Parsers/02_EJQTDE_spider}/ejqtde_save.py (100%) rename {02_EJQTDE_spider => Parsers/02_EJQTDE_spider}/ejqtde_scrawler.py (100%) rename {03_ProjectEuclid_spider => Parsers/03_ProjectEuclid_spider}/projecteuclid_main (100%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_detail.py (100%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_link.py (100%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_main.py (100%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_save.py (100%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_scrawl.py (92%) rename {04_SpringerOpen_spider => Parsers/04_SpringerOpen_spider}/SD_threads.py (100%) rename {05_WorldScientific_spider => Parsers/05_WorldScientific_spider}/WorldScientific (100%) diff --git a/00_Data/Merged/EJQTDE_buffer.zip b/Data/Merged/EJQTDE_buffer.zip similarity index 100% rename from 00_Data/Merged/EJQTDE_buffer.zip rename to Data/Merged/EJQTDE_buffer.zip diff --git a/00_Data/Merged/ejde_buffer.zip b/Data/Merged/ejde_buffer.zip similarity index 100% rename from 00_Data/Merged/ejde_buffer.zip rename to Data/Merged/ejde_buffer.zip diff --git a/00_Data/Origin/EJQTDE_buffer.zip b/Data/Origin/EJQTDE_buffer.zip similarity index 100% rename from 00_Data/Origin/EJQTDE_buffer.zip rename to Data/Origin/EJQTDE_buffer.zip diff --git a/00_Data/Origin/ejde_buffer.zip b/Data/Origin/ejde_buffer.zip similarity index 100% rename from 00_Data/Origin/ejde_buffer.zip rename to Data/Origin/ejde_buffer.zip diff --git a/00_File_merge/Division_byName.py b/FileMerger/Division_byName.py similarity index 100% rename from 00_File_merge/Division_byName.py rename to FileMerger/Division_byName.py diff --git a/00_File_merge/Merge.py b/FileMerger/Merge.py similarity index 100% rename from 00_File_merge/Merge.py rename to FileMerger/Merge.py diff --git a/00_File_merge/fileReader.py b/FileMerger/fileReader.py similarity index 100% rename from 00_File_merge/fileReader.py rename to FileMerger/fileReader.py diff --git a/01_EJDE_spider/get_paper_citation.py b/Parsers/00_Citation_spider/get_paper_citation.py similarity index 100% rename from 01_EJDE_spider/get_paper_citation.py rename to Parsers/00_Citation_spider/get_paper_citation.py diff --git a/01_EJDE_spider/ejde_main.py b/Parsers/01_EJDE_spider/ejde_main.py similarity index 100% rename from 01_EJDE_spider/ejde_main.py rename to Parsers/01_EJDE_spider/ejde_main.py diff --git a/01_EJDE_spider/ejde_save.py b/Parsers/01_EJDE_spider/ejde_save.py similarity index 100% rename from 01_EJDE_spider/ejde_save.py rename to Parsers/01_EJDE_spider/ejde_save.py diff --git a/02_EJQTDE_spider/ejqtde_main.py b/Parsers/02_EJQTDE_spider/ejqtde_main.py similarity index 100% rename from 02_EJQTDE_spider/ejqtde_main.py rename to Parsers/02_EJQTDE_spider/ejqtde_main.py diff --git a/02_EJQTDE_spider/ejqtde_save.py b/Parsers/02_EJQTDE_spider/ejqtde_save.py similarity index 100% rename from 02_EJQTDE_spider/ejqtde_save.py rename to Parsers/02_EJQTDE_spider/ejqtde_save.py diff --git a/02_EJQTDE_spider/ejqtde_scrawler.py b/Parsers/02_EJQTDE_spider/ejqtde_scrawler.py similarity index 100% rename from 02_EJQTDE_spider/ejqtde_scrawler.py rename to Parsers/02_EJQTDE_spider/ejqtde_scrawler.py diff --git a/03_ProjectEuclid_spider/projecteuclid_main b/Parsers/03_ProjectEuclid_spider/projecteuclid_main similarity index 100% rename from 03_ProjectEuclid_spider/projecteuclid_main rename to Parsers/03_ProjectEuclid_spider/projecteuclid_main diff --git a/04_SpringerOpen_spider/SD_detail.py b/Parsers/04_SpringerOpen_spider/SD_detail.py similarity index 100% rename from 04_SpringerOpen_spider/SD_detail.py rename to Parsers/04_SpringerOpen_spider/SD_detail.py diff --git a/04_SpringerOpen_spider/SD_link.py b/Parsers/04_SpringerOpen_spider/SD_link.py similarity index 100% rename from 04_SpringerOpen_spider/SD_link.py rename to Parsers/04_SpringerOpen_spider/SD_link.py diff --git a/04_SpringerOpen_spider/SD_main.py b/Parsers/04_SpringerOpen_spider/SD_main.py similarity index 100% rename from 04_SpringerOpen_spider/SD_main.py rename to Parsers/04_SpringerOpen_spider/SD_main.py diff --git a/04_SpringerOpen_spider/SD_save.py b/Parsers/04_SpringerOpen_spider/SD_save.py similarity index 100% rename from 04_SpringerOpen_spider/SD_save.py rename to Parsers/04_SpringerOpen_spider/SD_save.py diff --git a/04_SpringerOpen_spider/SD_scrawl.py b/Parsers/04_SpringerOpen_spider/SD_scrawl.py similarity index 92% rename from 04_SpringerOpen_spider/SD_scrawl.py rename to Parsers/04_SpringerOpen_spider/SD_scrawl.py index 31dbee4..74ec86a 100644 --- a/04_SpringerOpen_spider/SD_scrawl.py +++ b/Parsers/04_SpringerOpen_spider/SD_scrawl.py @@ -44,10 +44,10 @@ def Scrawl(Link): # 放入json文件暂存（小文件） if Article_data: index = str(uuid.uuid4()) - SD_save.save_data(Article_data, "Article_TS", index + ".json") + SD_save.save_data(Article_data, "Article_TS", index + ".json") print('Finished: ' + index + ' has been added to ./SpringerOpen_buffer/Article_TS/') if Author_data: index = str(uuid.uuid4()) - SD_save.save_data(Author_data, "Author_TS", index + ".json") + SD_save.save_data(Author_data, "Author_TS", index + ".json") print('Finished: ' + index + ' has been added to ./SpringerOpen_buffer/Author_TS/') \ No newline at end of file diff --git a/04_SpringerOpen_spider/SD_threads.py b/Parsers/04_SpringerOpen_spider/SD_threads.py similarity index 100% rename from 04_SpringerOpen_spider/SD_threads.py rename to Parsers/04_SpringerOpen_spider/SD_threads.py diff --git a/05_WorldScientific_spider/WorldScientific b/Parsers/05_WorldScientific_spider/WorldScientific similarity index 100% rename from 05_WorldScientific_spider/WorldScientific rename to Parsers/05_WorldScientific_spider/WorldScientific