From d374a3766bed131c24762e72d8c062f7d7dc345f Mon Sep 17 00:00:00 2001 From: Chenxiao Xia Date: Tue, 12 Sep 2023 09:05:55 +0800 Subject: [PATCH] Update new save code --- 01_EJDE_spider/ejde_save.py | 13 ++++--------- 02_EJQTDE_spider/ejqtde_save.py | 12 ++++-------- 04_SpringerOpen_spider/SD_save.py | 14 +++++--------- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/01_EJDE_spider/ejde_save.py b/01_EJDE_spider/ejde_save.py index eee016b..7ca56d9 100644 --- a/01_EJDE_spider/ejde_save.py +++ b/01_EJDE_spider/ejde_save.py @@ -30,24 +30,19 @@ def Transf(): file_path = os.path.join(folder_path, filename) with open(file_path, 'r', encoding='utf-8') as file: data = json.load(file) - for Dict in data: if Dict.get('volume') is not None or Dict.get('affiliation', [{}])[0].get('year', 0) is not None: # Select data - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009): + if int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009: data_oldest.append(Dict) - if (isinstance(Dict, dict) and 2010 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014: data_2010_2014.append(Dict) - if (isinstance(Dict, dict) and 2015 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020: data_2015_2020.append(Dict) - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) >= 2021): + else: data_newest.append(Dict) # Transfer diff --git a/02_EJQTDE_spider/ejqtde_save.py b/02_EJQTDE_spider/ejqtde_save.py index 6a7b8b0..e167184 100644 --- a/02_EJQTDE_spider/ejqtde_save.py +++ b/02_EJQTDE_spider/ejqtde_save.py @@ -36,20 +36,16 @@ def Transf(): for Dict in data: if Dict.get('volume') is not None or Dict.get('affiliation', [{}])[0].get('year', 0) is not None: # Select data - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009): + if int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009: data_oldest.append(Dict) - if (isinstance(Dict, dict) and 2010 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014: data_2010_2014.append(Dict) - if (isinstance(Dict, dict) and 2015 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020: data_2015_2020.append(Dict) - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) >= 2021): + else: data_newest.append(Dict) # Transfer diff --git a/04_SpringerOpen_spider/SD_save.py b/04_SpringerOpen_spider/SD_save.py index a70ef86..329f9fb 100644 --- a/04_SpringerOpen_spider/SD_save.py +++ b/04_SpringerOpen_spider/SD_save.py @@ -33,21 +33,17 @@ def Transf(): for Dict in data: if Dict.get('volume') is not None or Dict.get('affiliation', [{}])[0].get('year', 0) is not None: - # 筛选文章 - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009): + # Select data + if int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2009: data_oldest.append(Dict) - if (isinstance(Dict, dict) and 2010 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2014: data_2010_2014.append(Dict) - if (isinstance(Dict, dict) and 2015 <= int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020): + elif int(Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) <= 2020: data_2015_2020.append(Dict) - if (isinstance(Dict, dict) and int( - Dict.get('volume') or Dict.get('affiliation', [{}])[0].get('year', 0)) >= 2021): + else: data_newest.append(Dict) # 转存