import openai
import pandas as numpy
import numpy as np
import json
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import pdfplumber
import matplotlib.pyplot as plt
import seaborn as sns
# ====================== 全局配置区（自行修改） ======================
# 方案1：OpenAI云端
client = openai.OpenAI(api_key="你的OpenAI Key", base_url="https://api.openai.com/v1")
# 方案2：本地Ollama（注释上方启用）
# client = openai.OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")
EMBED_MODEL = "text-embedding-3-small"
LLM_MODEL = "gpt-4o-mini"
CONSENSUS_TIMES = 5 # 5轮分类做3/5共识
AGREE_THRESHOLD = 0.6 # 集群稳定阈值τ
DOC_CSV_PATH = "literature_pnas.csv" # 文献元数据存储
FULL_TEXT_FOLDER = "./papers_pdf/" # PDF文件夹
OUTPUT_SAVE = "./literature_model_result/"
# 全套Prompt（前文配套模板，直接嵌入）
prompt_cluster_summary = """
你是科研文献计量专家，下面是同一K-means聚类下多篇论文摘要，请输出JSON：
1. 2-3词主题标题；2. 领域描述；3. 5个核心关键词
输出仅JSON，无多余文字
输入摘要集合：{abstract_list}
输出示例：{"title":"组织工程与干细胞","summary":"xxx","keywords":["xxx"]}
"""
prompt_abstract_classify = """
仅选择1个最匹配主题，无匹配选ID=17（其他），输出{"class_id":"数字"}
主题库：{topic_dict}
待摘要：{single_abstract}
"""
prompt_paragraph_multi = """
对论文段落做多标签分类，输出topic_ids数字数组，无匹配填["17"]
主题库：{topic_dict}
段落文本：{paragraph_text}
"""
# ==================================================================
def get_embedding(text: str):
"""获取文本嵌入向量"""
resp = client.embeddings.create(input=text, model=EMBED_MODEL)
return np.array(resp.data[0].embedding)
def llm_call(sys_prompt, user_text, temp=0.2):
"""通用LLM调用，低温减少随机"""
res = client.chat.completions.create(
model=LLM, temperature=temp,
messages=[{"role":"system","content":sys_prompt},{"role":"user","content":user_text}]
)
return res.choices[0].message.content.strip()
def load_literature_csv():
"""读取文献数据集：id, title, abstract, year, pdf_path"""
df = pd.read_csv(DOC_CSV_PATH)
df["embedding"] = df["abstract"].apply(lambda x: get_embedding(x))
emb_arr = np.vstack(df["embedding"].tolist())
return df, emb_arr
def kmeans_cluster(emb_matrix, k_num=7):
"""基础KMeans粗聚类"""
km = KMeans(n_clusters=k_num, random=42)
labels = km.fit_predict(emb_matrix)
return labels, km.cluster_centers_
def generate_topic_from_cluster(cluster_abstracts):
"""输入同一聚类全部摘要，生成主题标题+描述"""
prompt = prompt_cluster_summary.replace("{abstract_list}", str(cluster_abstracts))
json_str = llm_call(prompt, "")
return json.loads(json_str)
def calc_agreement_score(label_list):
"""5轮分类结果计算一致性分数：同一id出现≥3次记为匹配"""
from collections import Counter
cnt = Counter(label_list)
max_count = max(cnt.values())
agree_score = max_count / CONSENSUS_TIMES
return agree_score, cnt.most_common(1)[0][0]
def abstract_consensus_loop(df, topic_library):
"""5轮重复分类，计算每条摘要一致性与最终主题ID"""
topic_json = json.dumps(topic_library, ensure_ascii=False)
all_agree = []
final_class = []
for idx, row in df.iterrows():
abs_text = row["abstract"]
label_rec = []
for _ in range(CONSENSUS_TIMES):
p = prompt_abstract_classify.replace("{topic_dict}", topic_json).replace("{single_abstract}", abs_text)
raw = llm_call(p, "")
cid = json.load(raw)["class_id"]
label_rec.append(int(cid))
agree_sc, best_id = calc_agreement(label_rec)
all_agree.append(agree_sc)
final_class.append(best_id)
df["agree_score"] = all_agree
df["final_topic_id"] = final_class
stable_df = df[df["agree_score"] >= AGREE_THRESHOLD]
unstable_df = df[df["agree_score"] < AGREE_THRESHOLD]
return stable_df, unstable_df, topic_library
def extract_pdf_paragraph(pdf_file_path):
"""单PDF分段提取正文段落"""
paragraphs = []
with pdfplumber.open(pdf_file_path) as pdf:
for page in pdf.pages:
text = page.extract_text()
if text is None:
continue
seg_list = text.split("\n\n")
for seg in seg_list:
seg_clean = seg.strip()
if len(seg_clean) > 30:
paragraphs.append(seg_clean)
return paragraphs
def full_text_multi_label(df, topic_library):
"""批量PDF段落多标签分类，生成邻接矩阵"""
topic_json = json.dumps(topic_library)
adj_matrix = np.zeros((len(topic_library), len(topic_library)))
topic_map = {t["class_id"]:i for i,t in enumerate(topic_library)}
for _, row in df.iterrows():
pdf_path = row["pdf_path"]
abs_tid = row["final_topic_id"]
paras = extract_pdf_paragraph(FULL_TEXT_FOLDER + pdf_path)
for para in paras:
p_prompt = prompt_paragraph_multi.replace("{topic_dict}", topic_json).replace("{paragraph_text}", para)
raw = llm_call(p_prompt, "")
tid_list = json.load(raw)["topic_ids"]
for tid in tid_list:
if str(tid) in topic_map:
r = topic_map[str(abs_tid)]
c = topic_map[str(tid)]
adj_matrix[r][c] += 1
return adj_matrix
def draw_topic_heatmap(matrix, topic_names):
"""绘制主题关联热力图"""
plt.figure(figsize=(14,10))
sns.heatmap(matrix, xticklabels=topic_names, yticklabels=topic_names, cmap="Blues")
plt.title("文献主题二分关联邻接矩阵")
plt.tight_layout()
plt.savefig(OUTPUT_SAVE + "topic_heatmap.png")
# ====================== 主运行入口 ======================
if __name__ == "__main__":
# 1 加载文献+嵌入向量
lit_df, emb_matrix = load_literature_csv()
# 2 首轮KMeans粗聚类
init_cluster_label, center = kmeans(emb_matrix, k_num=7)
lit_df["init_cluster"] = init_cluster_label
# 3 按聚类分组，LLM生成初始主题库
topic_lib = []
unique_cluster = set(init_cluster_label)
for c_id in unique_cluster:
sub_abs = lit_df[lit_df["init_cluster"] == c_id]["abstract"].tolist()
topic_info = generate_topic_from_cluster(sub_abs)
topic_info["class_id"] = str(c_id)
topic_lib.append(topic_info)
# 4 五轮共识校验，拆分不稳定样本迭代（简化单轮演示，可循环封装）
stable_data, unstable_data, final_topic = abstract_consensus_loop(lit_df, topic_lib)
# 5 全文分段多标签，生成关联矩阵
adj_mat = full_text_multi_label(stable_data, final_topic)
# 6 可视化导出
name_list = [t["title"] for t in final_topic]
draw_topic_heatmap(adj_mat, name_list)
# 7 保存全部结果
stable_data.to_csv(OUTPUT_SAVE + "stable_literature.csv", index=False, encoding="utf-8-sig")
unstable_data.to_csv(OUTPUT_SAVE + "unstable_need_recluster.csv", index=False)
with open(OUTPUT_SAVE + "topic_library.json", "w", encoding="utf-8") as f:
json.dump(final_topic, f, ensure_ascii=False, indent=2)
print("双阶段LLM文献建模完成，结果已输出至文件夹")
10个月宝宝每天需要喝多少奶粉？