大模型实战｜企业知识库问答系统构建全流程：从数据接入到RAG架构落地

用户提问 --> 问题预处理 --> 检索相似文档（向量检索） --> 构建Prompt --> 调用LLM生成回答 --> 返回用户

from unstructured.partition.auto import partition
from pathlib import Path

def extract_text(file_path):
    elements = partition(filename=file_path)
    return "\n".join([el.text for el in elements if el.text.strip() != ""])

text = extract_text("example_docs/财务制度.pdf")

from bs4 import BeautifulSoup
import requests

def scrape_clean_text(url):
    resp = requests.get(url)
    soup = BeautifulSoup(resp.text, 'html.parser')
    [s.extract() for s in soup(['script', 'style', 'footer', 'nav'])]
    return soup.get_text(strip=True)

html_text = scrape_clean_text("https://intranet.company.com/rules")

from openai import OpenAI
import numpy as np

client = OpenAI(api_key="你的API_KEY")

def get_embedding(text):
    res = client.embeddings.create(
        input=text,
        model="text-embedding-3-small"
    )
    return np.array(res.data[0].embedding)

import faiss

dimension = 1536  # 向量维度，取决于embedding模型
index = faiss.IndexFlatL2(dimension)

# 假设我们有多个文本段的向量
vectors = np.array([get_embedding(chunk) for chunk in documents])
index.add(vectors)

def retrieve_docs(query, k=5):
    query_vec = get_embedding(query).astype('float32')
    D, I = index.search(query_vec.reshape(1, -1), k)
    return [documents[i] for i in I[0]]

def build_prompt(question, docs):
    context = "\n".join([f"文档片段{i+1}：{doc}" for i, doc in enumerate(docs)])
    return f"""你是一个企业内部的知识助手，请根据以下内容回答问题：

{context}

问题：{question}
请用简洁、准确的语言回答："""

prompt = build_prompt("公司财务报销流程是怎样的？", retrieve_docs("财务报销流程"))

res = client.chat.completions.create(
    model="gpt-4",
    messages=[{"role": "user", "content": prompt}]
)
print(res.choices[0].message.content)

import streamlit as st

st.title("企业知识库问答系统")

query = st.text_input("请输入你的问题：")

if query:
    docs = retrieve_docs(query)
    prompt = build_prompt(query, docs)
    answer = client.chat.completions.create(
        model="gpt-4",
        messages=[{"role": "user", "content": prompt}]
    ).choices[0].message.content
    st.write("### 回答：")
    st.write(answer)