embedding做HDBSCAN

目标,发掘文本噪音

import numpy as np
import hdbscan
import matplotlib.pyplot as plt
import seaborn as sns
import umap

# 1. 读取 embedding 数据
X = np.load("your_embedding.npy")  # 替换成你的路径

# 2. 使用 HDBSCAN 进行聚类
clusterer = hdbscan.HDBSCAN(min_cluster_size=30, min_samples=10)
cluster_labels = clusterer.fit_predict(X)
probabilities = clusterer.probabilities_

# 3. 提取噪声样本(label = -1)
noise_mask = (cluster_labels == -1)
noise_indices = np.where(noise_mask)[0]
print(f"识别出噪声样本数量: {len(noise_indices)} / {len(X)}")

# 4. 可选:找出代表性样本(高置信度)
representative_indices = probabilities.argsort()[::-1][:100]

# 5. 用 UMAP(余弦距离)降维可视化
reducer = umap.UMAP(n_neighbors=15, min_dist=0.1)
X_2d = reducer.fit_transform(X)

# 6. 绘图
plt.figure(figsize=(10, 6))
sns.scatterplot(x=X_2d[:, 0], y=X_2d[:, 1], hue=cluster_labels, palette='tab20', s=10, legend=None)
plt.title("HDBSCAN 聚类 + 噪声识别(使用余弦距离)")
plt.show()
``` from bertopic import BERTopic from sentence_transformers import SentenceTransformer from umap import UMAP from hdbscan import HDBSCAN from bertopic.vectorizers import ClassTfidfTransformer import plotly.io as pio import pandas as pd from sklearn.feature_extraction.text import CountVectorizer from bertopic.representation import KeyBERTInspired data = pd.read_excel("数据.xlsx") # Step 1 - Embed documents embedding_model = SentenceTransformer('all-MiniLM-L12-v2') # Step 2 - Reduce dimensionality降维 # umap_model = UMAP(n_neighbors=15, n_components=5,min_dist=0.0, metric='cosine') umap_model = UMAP(n_neighbors=15, n_components=5,min_dist=0.0, metric='cosine', random_state=28) # Step 3 - Cluster reduced embeddings对降维向量聚类 hdbscan_model = HDBSCAN(min_cluster_size=15, metric='euclidean', prediction_data=True) # Step 4 - Create topic representation创造主题候选词 vectorizer_model = CountVectorizer(stop_words=None) # vectorizer_model = CountVectorizer(stop_words=["人工智能","ai","AI"]) # Step 5 - Create topic representation ctfidf_model = ClassTfidfTransformer() # Step 6 - (Optional) Fine-tune topic representations with a `bertopic.representation` model representation_model = KeyBERTInspired() # 训练bertopic主题模型 topic_model = BERTopic( embedding_model=embedding_model, # Step 1 - Extract embeddings umap_model=umap_model, # Step 2 - Reduce dimensionality hdbscan_model=hdbscan_model, # Step 3 - Cluster reduced embeddings vectorizer_model=vectorizer_model, # Step 4 - Tokenize topics ctfidf_model=ctfidf_model, # Step 5 - Extract topic words representation_model=representation_model, # Step 6 - (Optional) Fine-tune topic representations ) # 使用fit_transform对输入文本向量化,然后使用topic_model模型提取主题topics,并且计算主题文档概率probabilities filtered_text = data["内容"].astype(str).tolist() topics, probabilities = topic_model.fit_transform(filtered_text) document_info = topic_model.get_document_info(filtered_text) print(document_info) # 查看每个主题数量 topic_freq = topic_model.get_topic_freq() print(topic_freq) # 查看某个主题-词的概率分布 topic = topic_model.get_topic(0) print(topic) # 主题-词概率分布 pic_bar = topic_model.visualize_barchart() pio.show(pic_bar) # 文档主题聚类 embeddings = embedding_model.encode(filtered_text, show_progress_bar=False) pic_doc = topic_model.visualize_documents(filtered_text, embeddings=embeddings) pio.show(pic_doc) # 聚类分层 pic_hie = topic_model.visualize_hierarchy() pio.show(pic_hie) # 主题相似度热力图 pic_heat = topic_model.visualize_heatmap() pio.show(pic_heat) # 主题模排名图 pic_term_rank = topic_model.visualize_term_rank() pio.show(pic_term_rank) # 隐含主题主题分布图 pic_topics = topic_model.visualize_topics() pio.show(pic_topics) #DTM图 summary=data['内容'].astype(str).tolist() timepoint = data['时间'].tolist() timepoint = pd.Series(timepoint) print(timepoint[:10]) topics_over_time = topic_model.topics_over_time(summary, timepoint, datetime_format='mixed', nr_bins=20, evolution_tuning=True) DTM = topic_model.visualize_topics_over_time(topics_over_time, title='DTM',) pio.show(DTM)```请解释这个代码内容
03-13
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值