一天一个Python知识点——Day 164:词嵌入技术

原始：I absolutely LOVED the movie!!! It was soooo amazing 😍😍😍 #mustwatch处理后：['absolutely', 'love', 'movie', 'amazing', 'mustwatch']

movie → [0,0,0,1,0,0,...]  (假设词表大小10000，movie是第4个词)love  → [0,1,0,0,0,0,...]

输入：中心词 "coffee" → 预测上下文 ["I", "drank", "a", "cup", "of"]

from gensim.models import Word2Vecfrom nltk.tokenize import word_tokenizeimport nltknltk.download('punkt')# 准备语料（已经预处理的句子列表）sentences = [    "I love natural language processing".split(),    "Word embeddings are amazing".split(),    "I love coffee and tea".split(),    "Coffee and tea are both beverages".split()]# 训练Word2Vec模型model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)# 查看词向量vector = model.wv['coffee']print(f"'coffee'的向量（前10维）：{vector[:10]}")# 找相似词similar = model.wv.most_similar('coffee')print("与'coffee'最相似的词：", similar)# 词类比：国王 - 男人 + 女人 ≈ 王后result = model.wv.most_similar(positive=['woman', 'king'], negative=['man'])print("国王 - 男人 + 女人 ≈", result[0][0])

from gensim.scripts.glove2word2vec import glove2word2vecfrom gensim.models import KeyedVectors# 将GloVe格式转为Word2Vec格式glove_input_file = 'glove.6B.100d.txt'word2vec_output_file = 'glove.6B.100d.word2vec.txt'glove2word2vec(glove_input_file, word2vec_output_file)# 加载模型model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)# 使用similar = model.most_similar('coffee')print("GloVe中与'coffee'最相似的词：", similar)

"coffee" → <co, coff, offe, ffee, fee> 等字符n-gram

from gensim.models import FastText# 训练FastTextmodel = FastText(sentences, vector_size=100, window=5, min_count=1, workers=4)# 对于未登录词，仍能得到向量vector_oov = model.wv['never_seen_word']print("未登录词向量（前10维）：", vector_oov[:10])# 相似词查找similar = model.wv.most_similar('coffee')print("FastText中与'coffee'最相似的词：", similar)

import numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.linear_model import LogisticRegressionfrom sklearn.metrics import accuracy_scoreimport gensim.downloader as api# 加载预训练的GloVe词向量（第一次运行会下载）print("加载GloVe模型...")glove = api.load("glove-twitter-25")  # 25维，Twitter语料# 示例数据：简单情感句子texts = [    "I love this movie",    "This film is great",    "Terrible waste of time",    "I hate it",    "Absolutely wonderful",    "Worst movie ever",    "Pretty good actually",    "Not bad at all"]labels = [1, 1, 0, 0, 1, 0, 1, 1]  # 1:正面 0:负面def sentence_to_vector(sentence, model):    """将句子中所有词的词向量平均，作为句子向量"""    words = sentence.lower().split()    vectors = [model[word] for word in words if word in model]    if len(vectors) == 0:        return np.zeros(model.vector_size)    return np.mean(vectors, axis=0)# 转换为特征向量X = np.array([sentence_to_vector(s, glove) for s in texts])y = np.array(labels)# 划分训练集和测试集X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)# 训练分类器clf = LogisticRegression()clf.fit(X_train, y_train)# 评估y_pred = clf.predict(X_test)print("准确率：", accuracy_score(y_test, y_pred))

import matplotlib.pyplot as pltfrom sklearn.decomposition import PCA# 选择一些词words = ['coffee', 'tea', 'milk', 'water', 'juice',          'dog', 'cat', 'fish', 'bird', 'rabbit']# 获取向量vectors = np.array([glove[word] for word in words if word in glove])# PCA降维到2维pca = PCA(n_components=2)vectors_2d = pca.fit_transform(vectors)# 绘制plt.figure(figsize=(8,6))for i, word in enumerate(words):    if word in glove:        plt.scatter(vectors_2d[i,0], vectors_2d[i,1])        plt.annotate(word, (vectors_2d[i,0], vectors_2d[i,1]))plt.title("词向量PCA可视化")plt.show()

句子1："He deposited money in the bank."句子2："He sat on the river bank."