一个字段，搜遍所有媒体：用 semantic 字段为微信打造多模态搜索

📁 微信本地文件          ⚙️ 预处理脚本           📤 PUT _doc            🧠 Elasticsearch 9.5
图片 / 视频 / 音频  →   压缩 · Base64 编码  →   结构化 JSON        →   semantic 字段自动 embedding

🔍 用户查询              🌐 搜索 Web App         match / knn 查询       📊 结果排序
文字 / 图片 / 视频 / 音频  →  Python HTTP Server  →  ES 自动生成 query vector  →  相似度 > 60% 高亮展示

# 验证推理端点已就绪
GET _inference/embedding/.jina-embeddings-v5-omni-small

{
  "mappings": {
    "properties": {
      "embedding": {
        "type": "semantic",
        "inference_id": ".jina-embeddings-v5-omni-small"
      },
      "file_path":   { "type": "keyword" },
      "file_name":   { "type": "keyword" },
      "media_type":  { "type": "keyword" },
      "date_folder": { "type": "keyword" },
      "file_size":   { "type": "long"    },
      "indexed_at":  { "type": "date"    }
    }
  }
}

def process_image(path: Path) -> tuple[bool, str]:
    raw  = path.read_bytes()
    # 超过 900 KB 自动缩小，保持语义不变
    data = raw if len(raw) <= MAX_BINARY else resize_image(path)
    b64  = base64.b64encode(data).decode()
    mime = "image/png" if path.suffix.lower() == ".png" else "image/jpeg"

    doc = {
        "embedding": {          # ← ES 自动生成向量，无需手动调推理 API
            "type":  "image",
            "value": f"data:{mime};base64,{b64}"
        },
        "file_path":  str(path),
        "file_name":  path.name,
        "media_type": "image",
        "indexed_at": datetime.utcnow().isoformat() + "Z",
    }
    es_request("PUT", f"/{INDEX}/_doc/{path.stem}", doc)

def compress_video(path: Path) -> bytes:
    tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
    subprocess.run([
        "ffmpeg", "-y", "-i", str(path),
        "-vf", "scale='min(320,iw)':-2",
        "-crf", "40", "-preset", "fast",
        "-t", "30", "-an",     # 去音频，取前 30 秒
        tmp.name,
    ], capture_output=True, check=True)
    return Path(tmp.name).read_bytes()

# 索引时结构与图片完全一致，只改 type
doc["embedding"] = {
    "type":  "video",
    "value": f"data:video/mp4;base64,{b64}"
}

def compress_audio(path: Path) -> bytes:
    tmp = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
    subprocess.run([
        "ffmpeg", "-y", "-i", str(path),
        "-ar", "16000",   # 16 kHz 采样率
        "-ac", "1",       # 单声道
        "-b:a", "32k",    # 32 kbps 码率
        "-t", "120",      # 最长 2 分钟
        tmp.name,
    ], capture_output=True, check=True)
    return Path(tmp.name).read_bytes()

GET wechat-multimodal/_search
{
  "query": {
    "match": {
      "embedding": "海边风景，傍晚日落"
    }
  },
  "_source": { "excludes": ["embedding"] }
}

GET wechat-multimodal/_search
{
  "query": {
    "knn": {
      "field": "embedding",
      "query_vector_builder": {
        "embedding": {
          "input": {
            "type":  "image",
            "value": "data:image/jpeg;base64,<base64-encoded-bytes>"
          }
        }
      },
      "num_candidates": 100,
      "k": 20
    }
  }
}

GET wechat-multimodal/_search
{
  "query": {
    "knn": {
      "field": "embedding",
      "query_vector_builder": {
        "embedding": {
          "input": {
            "type":  "video",
            "value": "data:video/mp4;base64,<base64-encoded-bytes>"
          }
        }
      },
      "k": 10
    }
  }
}

GET wechat-multimodal/_search
{
  "query": {
    "knn": {
      "field": "embedding",
      "query_vector_builder": {
        "embedding": {
          "input": {
            "type":  "audio",
            "value": "data:audio/mpeg;base64,<base64-encoded-bytes>"
          }
        }
      },
      "k": 10
    }
  }
}

def text_search(query: str, size=20, filter_type=None):
    """文字查询：match 即可，ES 自动生成 embedding"""
    body = {
        "query": { "match": { "embedding": query } },
        "_source": SOURCE_FIELDS,
        "size": size,
    }
    if filter_type:
        body["query"] = {
            "bool": {
                "must":   { "match": { "embedding": query } },
                "filter": { "term": { "media_type": filter_type } }
            }
        }
    return es_request("POST", f"/{INDEX}/_search", body)


def media_search(media_type: str, data_uri: str, size=20):
    """媒体查询：knn + query_vector_builder，type 动态传入"""
    return es_request("POST", f"/{INDEX}/_search", {
        "query": {
            "knn": {
                "field": "embedding",
                "query_vector_builder": {
                    "embedding": {
                        "input": { "type": media_type, "value": data_uri }
                    }
                },
                "num_candidates": size * 5,
                "k": size,
            }
        },
        "_source": SOURCE_FIELDS,
        "size": size,
    })