开源模型落地正当时｜面向全领域技术人AI启动

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# 加载Qwen2.5-7B-Instruct
model_id = "Qwen/Qwen2.5-7B-Instruct"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,    # 半精度省显存
    device_map="auto",              # 自动分配到GPU
)

# 构造对话
messages = [
    {"role": "system", "content": "你是一个专业的技术助手，回答要准确、简洁。"},
    {"role": "user", "content": "解释一下什么是RAG？"},
]

text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(text, return_tensors="pt").to(model.device)

outputs = model.generate(
    **inputs,
    max_new_tokens=512,
    temperature=0.7,
    top_p=0.9,
)
response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print(response)

# 安装vLLM
pip install vllm

# 启动一个OpenAI兼容的API服务
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct \
    --port 8000 \
    --tensor-parallel-size 1 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 8192

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="empty",  # 本地部署不需要真实key
)

response = client.chat.completions.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    messages=[
        {"role": "system", "content": "你是技术助手"},
        {"role": "user", "content": "用一句话解释RAG"},
    ],
    temperature=0.7,
    max_tokens=256,
)

print(response.choices[0].message.content)

import os
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import DirectoryLoader, PyPDFLoader
from langchain_community.embeddings import HuggingFaceEmbeddings
from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType, utility

# ---- 1. 加载和分块文档 ----
loader = DirectoryLoader(
    "./docs",  # 存放企业文档的目录
    glob="**/*.pdf",
    loader_cls=PyPDFLoader,
)
documents = loader.load()

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,          # 每个块约512字符
    chunk_overlap=50,        # 块之间重叠50字符，保证语义连续
    separators=["\n\n", "\n", "。", "！", "？", "；", " ", ""],
)
chunks = text_splitter.split_documents(documents)
print(f"文档分块完成: {len(chunks)} 个文本块")

# ---- 2. 初始化Embedding模型 ----
embedding_model = HuggingFaceEmbeddings(
    model_name="BAAI/bge-m3",
    model_kwargs={"device": "cuda"},
)
texts = [chunk.page_content for chunk in chunks]
metadatas = [chunk.metadata for chunk in chunks]
embeddings = embedding_model.embed_documents(texts)

# ---- 3. 存入Milvus ----
connections.connect(host="localhost", port="19530")

# 定义集合Schema
fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
    FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=2048),
    FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=len(embeddings[0])),
    FieldSchema(name="source", dtype=DataType.VARCHAR, max_length=512),
]
schema = CollectionSchema(fields, description="企业知识库向量索引")

if utility.has_collection("knowledge_base"):
    utility.drop_collection("knowledge_base")
collection = Collection("knowledge_base", schema)

# 创建IVF索引
collection.create_index(
    "embedding",
    {
        "index_type": "IVF_FLAT",
        "metric_type": "IP",        # 内积，配合归一化的embedding
        "params": {"nlist": 128},
    }
)

# 批量插入
collection.insert([
    list(range(len(texts))),     # id由auto_id生成，这里占位
    texts,
    embeddings,
    [m.get("source", "unknown") for m in metadatas],
])
collection.load()
print(f"向量入库完成，共 {len(texts)} 条记录")

from openai import OpenAI
from langchain_community.embeddings import HuggingFaceEmbeddings
from pymilvus import Collection, connections
import json

class KnowledgeBaseQA:
    """企业知识库问答系统"""
    
    def __init__(self):
        # 连接向量数据库
        connections.connect(host="localhost", port="19530")
        self.collection = Collection("knowledge_base")
        self.collection.load()
        
        # Embedding模型（用于Query向量化）
        self.embedder = HuggingFaceEmbeddings(
            model_name="BAAI/bge-m3",
            model_kwargs={"device": "cuda"},
        )
        
        # LLM推理服务（vLLM启动的本地服务）
        self.llm_client = OpenAI(
            base_url="http://localhost:8000/v1",
            api_key="empty",
        )
    
    def retrieve(self, query: str, top_k: int = 5):
        """向量检索Top-K相关文档块"""
        query_vec = self.embedder.embed_query(query)
        
        results = self.collection.search(
            data=[query_vec],
            anns_field="embedding",
            param={"metric_type": "IP", "params": {"nprobe": 16}},
            limit=top_k,
            output_fields=["text", "source"],
        )
        
        docs = []
        for hit in results[0]:
            docs.append({
                "text": hit.entity.get_value("text"),
                "source": hit.entity.get_value("source"),
                "score": hit.score,
            })
        return docs
    
    def rerank(self, query: str, docs: list, top_n: int = 3):
        """简单重排：基于关键词重叠度做二次过滤
        生产环境建议用Cohere Rerank或bge-reranker模型"""
        query_terms = set(query)
        for doc in docs:
            doc_terms = set(doc["text"])
            overlap = len(query_terms & doc_terms)
            doc["rerank_score"] = doc["score"] * 0.7 + overlap * 0.01
        docs.sort(key=lambda x: x["rerank_score"], reverse=True)
        return docs[:top_n]
    
    def generate(self, query: str, retrieved_docs: list):
        """组装Prompt，调用LLM生成回答"""
        context = "\n\n".join([
            f"[来源{i+1}] {doc['source']}\n{doc['text']}"
            for i, doc in enumerate(retrieved_docs)
        ])
        
        system_prompt = """你是一个企业知识库助手。请根据以下检索到的资料回答用户问题。
规则：
1. 只基于提供的资料回答，不要编造信息
2. 如果资料中没有相关内容，明确说"根据现有资料，暂无相关信息"
3. 回答末尾标注引用来源编号
4. 回答要专业、准确、有条理"""

        user_prompt = f"""参考资料：
{context}

用户问题：{query}

请根据以上参考资料回答："""

        response = self.llm_client.chat.completions.create(
            model="Qwen/Qwen2.5-7B-Instruct",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt},
            ],
            temperature=0.3,       # 低温度保证回答稳定
            top_p=0.85,
            max_tokens=1024,
        )
        
        answer = response.choices[0].message.content
        sources = [doc["source"] for doc in retrieved_docs]
        return answer, sources
    
    def ask(self, query: str):
        """完整问答流程：检索 → 重排 → 生成"""
        # 1. 向量检索
        docs = self.retrieve(query, top_k=5)
        
        # 2. 重排序
        docs = self.rerank(query, docs, top_n=3)
        
        # 3. 生成回答
        answer, sources = self.generate(query, docs)
        
        return {
            "query": query,
            "answer": answer,
            "sources": sources,
        }


# ---- 使用示例 ----
if __name__ == "__main__":
    qa = KnowledgeBaseQA()
    
    result = qa.ask("公司的年假制度是怎样的？新员工第一年有多少天年假？")
    
    print(f"问题: {result['query']}")
    print(f"回答: {result['answer']}")
    print(f"引用来源: {result['sources']}")

# 方法1：降低gpu-memory-utilization，留出余量
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct \
    --gpu-memory-utilization 0.85 \
    --max-model-len 4096   # 别设太大，够用就行

# 方法2：上量化，用AWQ或GPTQ量化后的模型
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct-AWQ \
    --quantization awq \
    --gpu-memory-utilization 0.9

# RAG场景建议配置
response = client.chat.completions.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    messages=messages,
    temperature=0.1,    # 低温度=稳定输出
    top_p=0.85,
    max_tokens=1024,
    seed=42,            # 固定随机种子，保证可复现
)