一、Milvus 安装部署(Docker Compose)
1. 环境准备
- Docker 版本 ≥ 20.10.0
- Docker Compose 版本 ≥ 2.0.0
2. 部署步骤
# 1. 下载 Milvus 配置文件
wget https://github.com/milvus-io/milvus/releases/download/v2.3.5/milvus-standalone-docker-compose.yml -O docker-compose.yml
# 2. 启动 Milvus 服务
docker-compose up -d
# 3. 验证服务状态
docker-compose ps
# 预期输出:milvus-standalone 状态为 Up
3. 停止服务(可选)
docker-compose down
二、Java 客户端配置
1. 依赖引入(Maven)
在 pom.xml 中添加以下依赖:
<dependencies>
<!-- Milvus Java SDK -->
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.3.5</version>
</dependency>
<!-- 用于生成随机向量 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<!-- 日志框架 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.11</version>
</dependency>
</dependencies>
2. 核心配置类
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.ConnectParam;
public class MilvusConfig {
// Milvus 服务地址(默认端口 19530)
private static final String HOST = "localhost";
private static final int PORT = 19530;
// 单例模式创建 Milvus 客户端
private static MilvusServiceClient client;
public static MilvusServiceClient getClient() {
if (client == null) {
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(HOST)
.withPort(PORT)
.build();
client = new MilvusServiceClient(connectParam);
}
return client;
}
// 关闭客户端
public static void closeClient() {
if (client != null) {
client.close();
client = null;
}
}
}
三、核心功能实现
1. 集合管理(Collection)
创建集合
import io.milvus.param.*;
import io.milvus.param.collection.*;
public class CollectionManager {
private final MilvusServiceClient client;
public CollectionManager(MilvusServiceClient client) {
this.client = client;
}
// 创建集合
public void createCollection(String collectionName, int dimension) {
// 定义字段
FieldType idField = FieldType.newBuilder()
.withName("id")
.withDataType(DataType.Int64)
.withPrimaryKey(true)
.withAutoID(true) // 自动生成 ID
.build();
FieldType vectorField = FieldType.newBuilder()
.withName("vector")
.withDataType(DataType.FloatVector)
.withDimension(dimension) // 向量维度
.build();
FieldType textField = FieldType.newBuilder()
.withName("text")
.withDataType(DataType.VarChar)
.withMaxLength(512)
.build();
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
.withCollectionName(collectionName)
.withDescription("Java Milvus Demo Collection")
.addFieldType(idField)
.addFieldType(vectorField)
.addFieldType(textField)
.withEnableDynamicField(true) // 启用动态字段
.build();
R<RpcStatus> response = client.createCollection(createParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Create collection failed: " + response.getMessage());
}
System.out.println("Collection created successfully!");
}
// 检查集合是否存在
public boolean hasCollection(String collectionName) {
HasCollectionParam param = HasCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
R<Boolean> response = client.hasCollection(param);
return response.getData();
}
// 删除集合
public void dropCollection(String collectionName) {
DropCollectionParam param = DropCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
client.dropCollection(param);
System.out.println("Collection dropped successfully!");
}
}
2. 数据生成与插入
向量数据生成工具
import org.apache.commons.math3.random.RandomDataGenerator;
import java.util.ArrayList;
import java.util.List;
public class VectorGenerator {
// 生成随机浮点向量
public static List<List<Float>> generateRandomVectors(int count, int dimension) {
List<List<Float>> vectors = new ArrayList<>(count);
RandomDataGenerator random = new RandomDataGenerator();
for (int i = 0; i < count; i++) {
List<Float> vector = new ArrayList<>(dimension);
for (int j = 0; j < dimension; j++) {
vector.add((float) random.nextUniform(-1.0, 1.0));
}
vectors.add(vector);
}
return vectors;
}
// 生成测试文本数据
public static List<String> generateTestTexts(int count) {
List<String> texts = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
texts.add("Test text " + i);
}
return texts;
}
}
批量插入数据
import io.milvus.param.*;
import io.milvus.param.insert.*;
import io.milvus.response.*;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class DataInsertor {
private final MilvusServiceClient client;
public DataInsertor(MilvusServiceClient client) {
this.client = client;
}
// 批量插入数据
public long insertBatch(String collectionName, List<List<Float>> vectors, List<String> texts) {
InsertParam insertParam = InsertParam.newBuilder()
.withCollectionName(collectionName)
.withFields(List.of(
InsertParam.Field.newBuilder()
.withName("vector")
.withValues(vectors)
.build(),
InsertParam.Field.newBuilder()
.withName("text")
.withValues(texts)
.build()
))
.build();
R<MutationResult> response = client.insert(insertParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Insert failed: " + response.getMessage());
}
return response.getData().getInsertCount();
}
// 百万级数据批量插入(多线程)
public void insertMillionData(String collectionName, int dimension, int totalCount) {
final int batchSize = 10000; // 每批次插入 10,000 条
final int threadCount = 4; // 4 线程并行插入
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
long startTime = System.currentTimeMillis();
for (int i = 0; i < totalCount; i += batchSize) {
final int currentBatch = Math.min(batchSize, totalCount - i);
executor.submit(() -> {
List<List<Float>> vectors = VectorGenerator.generateRandomVectors(currentBatch, dimension);
List<String> texts = VectorGenerator.generateTestTexts(currentBatch);
insertBatch(collectionName, vectors, texts);
});
}
executor.shutdown();
try {
executor.awaitTermination(1, TimeUnit.HOURS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
long endTime = System.currentTimeMillis();
System.out.printf("Inserted %d vectors in %.2f seconds%n",
totalCount, (endTime - startTime) / 1000.0);
}
}
3. 索引创建与加载
索引管理
import io.milvus.param.*;
import io.milvus.param.index.*;
public class IndexManager {
private final MilvusServiceClient client;
public IndexManager(MilvusServiceClient client) {
this.client = client;
}
// 创建 IVF_FLAT 索引(适用于中小规模数据)
public void createIVFFlatIndex(String collectionName, int nlist) {
IndexParam indexParam = IndexParam.newBuilder()
.withCollectionName(collectionName)
.withFieldName("vector")
.withIndexName("vector_index")
.withIndexType(IndexType.IVF_FLAT)
.withMetricType(MetricType.L2) // 欧氏距离
.withExtraParam("{\"nlist\": " + nlist + "}")
.build();
R<RpcStatus> response = client.createIndex(indexParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Create index failed: " + response.getMessage());
}
System.out.println("IVF_FLAT index created successfully!");
}
// 创建 HNSW 索引(适用于大规模数据,检索更快)
public void createHNSWIndex(String collectionName, int M, int efConstruction) {
IndexParam indexParam = IndexParam.newBuilder()
.withCollectionName(collectionName)
.withFieldName("vector")
.withIndexName("vector_index")
.withIndexType(IndexType.HNSW)
.withMetricType(MetricType.L2)
.withExtraParam("{\"M\": " + M + ", \"efConstruction\": " + efConstruction + "}")
.build();
R<RpcStatus> response = client.createIndex(indexParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Create index failed: " + response.getMessage());
}
System.out.println("HNSW index created successfully!");
}
// 加载集合到内存
public void loadCollection(String collectionName) {
LoadCollectionParam loadParam = LoadCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
R<RpcStatus> response = client.loadCollection(loadParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Load collection failed: " + response.getMessage());
}
System.out.println("Collection loaded to memory!");
}
// 释放集合内存
public void releaseCollection(String collectionName) {
ReleaseCollectionParam releaseParam = ReleaseCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build();
client.releaseCollection(releaseParam);
System.out.println("Collection released from memory!");
}
}
4. 向量检索
检索实现
import io.milvus.param.*;
import io.milvus.param.search.*;
import io.milvus.response.*;
import java.util.List;
public class VectorSearcher {
private final MilvusServiceClient client;
public VectorSearcher(MilvusServiceClient client) {
this.client = client;
}
// 向量检索
public void searchVectors(String collectionName, List<Float> queryVector, int topK) {
// 构建检索参数
SearchParam searchParam = SearchParam.newBuilder()
.withCollectionName(collectionName)
.withMetricType(MetricType.L2)
.withOutFields(List.of("id", "text")) // 返回字段
.withTopK(topK)
.withVectors(List.of(queryVector))
.withVectorFieldName("vector")
.withParams("{\"nprobe\": 10}") // IVF 索引参数:检索的聚类中心数量
// .withParams("{\"ef\": 64}") // HNSW 索引参数:检索时的 ef 值
.build();
long startTime = System.currentTimeMillis();
R<SearchResults> response = client.search(searchParam);
long endTime = System.currentTimeMillis();
if (response.getStatus() != R.Status.Success.getCode()) {
throw new RuntimeException("Search failed: " + response.getMessage());
}
// 处理检索结果
SearchResultsData resultsData = response.getData().getResults();
System.out.printf("Search completed in %.2f ms%n", (endTime - startTime) / 1.0);
System.out.printf("Found %d results for top %d%n", resultsData.getRowCount(), topK);
// 输出结果
for (int i = 0; i < resultsData.getRowCount(); i++) {
SearchResultsData.RowRecord row = resultsData.getRowRecord(i);
long id = row.getLong("id");
String text = row.getString("text");
float distance = row.getFloat("distance");
System.out.printf("Rank %d: ID=%d, Text=%s, Distance=%.6f%n",
i + 1, id, text, distance);
}
}
// 批量检索(多查询向量)
public void batchSearch(String collectionName, List<List<Float>> queryVectors, int topK) {
SearchParam searchParam = SearchParam.newBuilder()
.withCollectionName(collectionName)
.withMetricType(MetricType.L2)
.withOutFields(List.of("id", "text"))
.withTopK(topK)
.withVectors(queryVectors)
.withVectorFieldName("vector")
.withParams("{\"nprobe\": 10}")
.build();
long startTime = System.currentTimeMillis();
R<SearchResults> response = client.search(searchParam);
long endTime = System.currentTimeMillis();
System.out.printf("Batch search completed in %.2f ms%n", (endTime - startTime) / 1.0);
System.out.printf("Total queries: %d%n", queryVectors.size());
}
}
四、百万级向量检索完整示例
1. 主程序入口
public class MilvusJavaDemo {
private static final String COLLECTION_NAME = "java_demo_collection";
private static final int DIMENSION = 128; // 向量维度
private static final int TOTAL_COUNT = 1000000; // 百万级数据
public static void main(String[] args) {
MilvusServiceClient client = MilvusConfig.getClient();
try {
// 1. 集合管理
CollectionManager collectionManager = new CollectionManager(client);
if (collectionManager.hasCollection(COLLECTION_NAME)) {
collectionManager.dropCollection(COLLECTION_NAME);
}
collectionManager.createCollection(COLLECTION_NAME, DIMENSION);
// 2. 插入百万级数据
DataInsertor dataInsertor = new DataInsertor(client);
System.out.println("Start inserting " + TOTAL_COUNT + " vectors...");
dataInsertor.insertMillionData(COLLECTION_NAME, DIMENSION, TOTAL_COUNT);
// 3. 创建索引
IndexManager indexManager = new IndexManager(client);
// 选择索引类型:IVF_FLAT 或 HNSW
indexManager.createIVFFlatIndex(COLLECTION_NAME, 1024); // nlist=1024
// indexManager.createHNSWIndex(COLLECTION_NAME, 16, 256); // M=16, efConstruction=256
// 4. 加载集合到内存
indexManager.loadCollection(COLLECTION_NAME);
// 5. 向量检索
VectorSearcher searcher = new VectorSearcher(client);
// 生成查询向量
List<Float> queryVector = VectorGenerator.generateRandomVectors(1, DIMENSION).get(0);
System.out.println("\nStart searching...");
searcher.searchVectors(COLLECTION_NAME, queryVector, 10);
// 6. 批量检索测试
System.out.println("\nStart batch search...");
List<List<Float>> batchQueryVectors = VectorGenerator.generateRandomVectors(100, DIMENSION);
searcher.batchSearch(COLLECTION_NAME, batchQueryVectors, 10);
} catch (Exception e) {
e.printStackTrace();
} finally {
MilvusConfig.closeClient();
}
}
}
2. 运行结果示例
Collection created successfully!
Start inserting 1000000 vectors...
Inserted 1000000 vectors in 45.23 seconds
IVF_FLAT index created successfully!
Collection loaded to memory!
Start searching...
Search completed in 12.56 ms
Found 10 results for top 10
Rank 1: ID=123456, Text=Test text 123456, Distance=0.001234
Rank 2: ID=789012, Text=Test text 789012, Distance=0.002345
...
Start batch search...
Batch search completed in 89.32 ms
Total queries: 100
五、性能优化建议
-
插入优化
- 批次大小:建议 10,000-50,000 条 / 批次
- 多线程并行插入:充分利用 CPU 资源
- 关闭自动刷新:插入完成后手动调用
flush方法
-
索引优化
- IVF_FLAT:nlist 建议设置为 sqrt (数据量),如百万级数据设置为 1024
- HNSW:M=16-64,efConstruction=200-500(值越大索引质量越高,但构建时间越长)
- 检索时调整
nprobe(IVF)或ef(HNSW)参数平衡速度与精度
-
检索优化
- 批量检索:减少网络往返次数
- 只返回必要字段:降低数据传输开销
- 使用连接池:复用客户端连接
六、常见问题与解决方案
| 问题 | 解决方案 |
|---|---|
| 连接超时 | 检查 Milvus 服务是否正常运行,网络是否通畅 |
| 内存不足 | 调整 JVM 堆内存大小,或减少批次插入量 |
| 索引构建失败 | 检查索引参数是否合法,确保数据已插入 |
| 检索结果为空 | 确认集合已加载到内存,查询向量维度匹配 |
七、扩展功能
- 分区管理:针对大规模数据,可按时间或业务维度创建分区
- 数据更新 / 删除:支持根据主键更新或删除数据
- 混合检索:结合向量检索与标量过滤(如
text LIKE '%keyword%') - 分布式部署:Milvus 支持集群部署,提高可用性与扩展性
八、总结
本示例完整演示了 Milvus 从安装部署到百万级向量检索的全流程,涵盖了集合管理、数据插入、索引创建、向量检索等核心功能。通过 Java SDK,开发者可以轻松集成 Milvus 到自己的应用中,实现高效的向量检索服务。
建议根据实际业务场景调整向量维度、索引类型、检索参数等,以达到最佳的性能表现。
参考资料:

2056

被折叠的 条评论
为什么被折叠?



