25 lines
859 B
Python
25 lines
859 B
Python
from concurrent.futures import ThreadPoolExecutor, as_completed, wait
|
|
import SD_scrawl
|
|
|
|
# ==========多线程处理==========
|
|
def Threads(Links, Article_data, Author_data):
|
|
executor = ThreadPoolExecutor(max_workers=20) # 进程池
|
|
|
|
# 进行多线程处理
|
|
futures = [executor.submit(SD_scrawl.Scrawl, Link, Article_data, Author_data) for Link in Links]
|
|
|
|
# max_iterations = 5 # 最多同时进行数
|
|
# iteration_count = 0 # 计数器
|
|
|
|
# 等待所有进程完成
|
|
for future in as_completed(futures):
|
|
try:
|
|
future.result()
|
|
# # 限制最大同时爬取数
|
|
# iteration_count += 1 # Increment the counter
|
|
# if iteration_count >= max_iterations:
|
|
# break
|
|
except Exception as e:
|
|
print("An error occurred:", str(e))
|
|
|
|
wait(futures) |