init commit

requirements.txt

@@ -0,0 +1,49 @@
+# Core dependencies
+# IMPORTANT: Install PyTorch with CUDA 12.8 support first using:
+# pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128
+# Then install remaining dependencies with: pip install -r requirements.txt
+
+# PyTorch - version will be satisfied by pre-installed nightly build
+# Minimum version for CUDA 12.8 support (will use nightly/2.3.0+ in practice)
+torch>=2.2.0
+torchvision>=0.17.0
+torchaudio>=2.2.0
+# HuggingFace Transformers
+transformers>=4.38.0
+# HuggingFace Datasets
+datasets>=2.14.0
+# HuggingFace Tokenizers
+tokenizers>=0.15.0
+numpy>=1.23.0
+scipy>=1.10.0
+scikit-learn>=1.3.0
+pandas>=1.5.0
+FlagEmbedding>=1.2.0
+# For CPU use faiss-cpu
+faiss-gpu>=1.7.4
+psutil>=5.9.0
+omegaconf>=2.3.0
+toml>=0.10.2
+tqdm>=4.64.0
+pyyaml>=6.0
+jsonlines>=3.1.0
+huggingface-hub>=0.19.0
+jieba>=0.42.1
+requests>=2.28.0
+# Optional/dev dependencies
+# For Huawei Ascend NPU support (install only if needed)
+# torch-npu>=1.11.0.post1; platform_system=="Linux" and platform_machine=="aarch64"
+# ascend-compiler; platform_system=="Linux" and platform_machine=="aarch64"
+# For experiment tracking (optional)
+wandb  # optional, for Weights & Biases logging
+# For visualization (optional)
+tensorboard  # optional, for TensorBoard logging
+# For testing and development
+pytest  # optional, for running tests
+# For ModelScope support (optional)
+modelscope  # optional, for ModelScope model hub
+sentence-transformers>=2.2.2
+accelerate>=0.25.0
+rank-bm25>=0.2.2
+# For logging and distributed
+loguru>=0.7.0