一天一个Python知识点——Day 161:NLTK基础

pip install nltk

import nltk# 下载常用数据（只需一次）nltk.download('punkt')      # 分词模型nltk.download('stopwords')  # 停用词表nltk.download('averaged_perceptron_tagger')  # 词性标注模型nltk.download('maxent_ne_chunker')  # 命名实体识别nltk.download('words')      # 词汇表nltk.download('vader_lexicon')  # 情感分析词典

nltk.download('all')  # 约3.5GB，谨慎使用

from nltk.tokenize import word_tokenize, sent_tokenizetext = "Natural Language Processing with NLTK is fun. Let's learn it step by step."# 分句sentences = sent_tokenize(text)print("句子：", sentences)# 分词words = word_tokenize(text)print("单词：", words)# 查看单词数量print("单词数：", len(words))

句子： ['Natural Language Processing with NLTK is fun.', "Let's learn it step by step."]单词： ['Natural', 'Language', 'Processing', 'with', 'NLTK', 'is', 'fun', '.', 'Let', "'s", 'learn', 'it', 'step', 'by', 'step', '.']单词数： 16

from nltk.corpus import stopwordsstop_words = set(stopwords.words('english'))words = ['Natural', 'Language', 'Processing', 'is', 'fun', '.']filtered = [w for w in words if w.lower() not in stop_words]print("去除停用词后：", filtered)# 输出：['Natural', 'Language', 'Processing', 'fun', '.']

from nltk.stem import PorterStemmerstemmer = PorterStemmer()words = ['running', 'runner', 'runs', 'easily', 'fairly']stems = [stemmer.stem(w) for w in words]print("词干：", stems)# 输出：['run', 'runner', 'run', 'easili', 'fairli']  # 注意easily变成了easili

from nltk.stem import WordNetLemmatizernltk.download('wordnet')lemmatizer = WordNetLemmatizer()words = ['running', 'runner', 'runs', 'better', 'geese']lemmas = [lemmatizer.lemmatize(w, pos='v') for w in words]  # pos指定词性print("词形还原：", lemmas)# 输出：['run', 'runner', 'run', 'better', 'geese']  # 动词变回原形，名词保留

from nltk import pos_tagtext = "I love natural language processing"words = word_tokenize(text)tagged = pos_tag(words)print("词性标注结果：")for word, tag in tagged:    print(f"{word}: {tag}")

I: PRP（人称代词）love: VBP（动词，非第三人称单数现在时）natural: JJ（形容词）language: NN（名词，单数）processing: NN（名词，单数）

from nltk import ne_chunk# 需要先下载模型：nltk.download('maxent_ne_chunker')sentence = "Apple Inc. is planning to open a new store in New York next month."words = word_tokenize(sentence)tagged = pos_tag(words)entities = ne_chunk(tagged)print(entities)  # 输出树状结构

for chunk in entities:    if hasattr(chunk, 'label'):  # 是命名实体        print(f"实体：{chunk.label()} -> {' '.join(c[0] for c in chunk)}")

实体：ORGANIZATION -> Apple Inc.实体：GPE -> New York

from nltk import FreqDisttext = "the cat in the hat sat on the mat with a cat"words = word_tokenize(text)fdist = FreqDist(words)print("最常见的词：", fdist.most_common(3))print("单词'the'出现次数：", fdist['the'])# 绘制频率分布图fdist.plot(10, cumulative=False)

from nltk.sentiment import SentimentIntensityAnalyzernltk.download('vader_lexicon')sid = SentimentIntensityAnalyzer()sentences = [    "This movie was absolutely amazing! I loved it.",    "The food was terrible and the service was even worse.",    "It's okay, nothing special."]for sentence in sentences:    scores = sid.polarity_scores(sentence)    print(f"\n句子：{sentence}")    print(f"情感分数：{scores}")    print(f"总体情感：{scores['compound']}")

句子：This movie was absolutely amazing! I loved it.情感分数：{'neg': 0.0, 'neu': 0.282, 'pos': 0.718, 'compound': 0.8887}总体情感：0.8887句子：The food was terrible and the service was even worse.情感分数：{'neg': 0.454, 'neu': 0.546, 'pos': 0.0, 'compound': -0.8271}

from nltk.corpus import gutenberg, brown, reuters# 查看古腾堡计划语料库print(gutenberg.fileids())# 加载《圣经》bible = gutenberg.words('bible-kjv.txt')print("圣经单词数：", len(bible))# 布朗语料库（按文体分类）print(brown.categories())news_text = brown.words(categories='news')print("新闻类单词数：", len(news_text))# 路透社语料库print(reuters.fileids()[:5])print(reuters.categories())

from nltk.corpus import movie_reviewsimport randomnltk.download('movie_reviews')documents = [(list(movie_reviews.words(fileid)), category)             for category in movie_reviews.categories()             for fileid in movie_reviews.fileids(category)]random.shuffle(documents)print("样本数：", len(documents))print("正样本示例：", documents[0][0][:10], documents[0][1])

from nltk import FreqDist# 收集所有词all_words = FreqDist(w.lower() for w in movie_reviews.words())word_features = list(all_words.keys())[:2000]def document_features(document):    document_words = set(document)    features = {}    for word in word_features:        features[f'contains({word})'] = (word in document_words)    return features

featuresets = [(document_features(doc), category) for (doc, category) in documents]train_set, test_set = featuresets[100:], featuresets[:100]

from nltk import NaiveBayesClassifierclassifier = NaiveBayesClassifier.train(train_set)

print("准确率：", nltk.classify.accuracy(classifier, test_set))# 查看最有信息量的特征classifier.show_most_informative_features(10)

准确率： 0.81Most Informative Features        contains(bad) = True              neg : pos    =     13.2 : 1.0        contains(amazing) = True            pos : neg    =     11.0 : 1.0        contains(worst) = True             neg : pos    =     10.8 : 1.0        contains(fantastic) = True          pos : neg    =     9.8 : 1.0

def predict_sentiment(text):    words = word_tokenize(text.lower())    features = document_features(words)    return classifier.classify(features)print(predict_sentiment("This movie is really great and exciting!"))print(predict_sentiment("Terrible film, waste of time."))