from concurrent.futures import ThreadPoolExecutor, as_completed, wait
import SD_scrawl

# ==========多线程处理==========
def Threads(Links, Article_data, Author_data):
    executor = ThreadPoolExecutor(max_workers=20)       # 进程池

    # 进行多线程处理
    futures = [executor.submit(SD_scrawl.Scrawl, Link, Article_data, Author_data) for Link in Links]

    # max_iterations = 5      # 最多同时进行数
    # iteration_count = 0     # 计数器

    # 等待所有进程完成
    for future in as_completed(futures):
        try:
            future.result()
        #     # 限制最大同时爬取数
        #     iteration_count += 1  # Increment the counter
        #     if iteration_count >= max_iterations:
        #         break
        except Exception as e:
            print("An error occurred:", str(e))

    wait(futures)