一天一个Python知识点——Day 163:文本预处理实战

"RT @user: This is SOOOO exciting!!! Can't wait for the #event 😍😍😍 http://t.co/xyz"

"这家店的东西质量很好！！！服务态度也超赞的～～但是价格有点小贵……下次还会来💯"

import redef clean_text(text):    """基础清洗函数"""    # 移除HTML标签    text = re.sub(r'<[^>]+>', '', text)    # 移除URL    text = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+])+', '', text)    # 移除@提及    text = re.sub(r'@\w+', '', text)    # 移除特殊符号（保留字母、数字、中文、基本标点）    text = re.sub(r'[^\w\s\u4e00-\u9fff。，！？、]', '', text)    # 合并多余空白    text = re.sub(r'\s+', ' ', text).strip()    return text# 示例dirty_text = "<p>RT @user: This is SOOOO exciting!!! Can't wait for the #event 😍😍😍 http://t.co/xyz</p>"clean = clean_text(dirty_text)print("清洗后：", clean)# 输出：RT This is SOOOO exciting Cant wait for the event

def normalize_text(text):    """文本统一化"""    # 转为小写（英文）    text = text.lower()    # 将数字替换为占位符    text = re.sub(r'\d+', '<NUM>', text)    # 将连续重复字母缩减（如 soooo → so）    text = re.sub(r'(.)\1{2,}', r'\1', text)    return text# 示例text = "This is SOOOO AMAZING!!! 12345"norm = normalize_text(text)print("统一后：", norm)# 输出：this is so amazing!!! <NUM>

import jiebaimport spacyfrom nltk.tokenize import word_tokenize# 英文分词（spaCy）nlp_en = spacy.load("en_core_web_sm")def tokenize_english(text):    doc = nlp_en(text)    return [token.text for token in doc]# 中文分词（jieba）def tokenize_chinese(text):    return list(jieba.cut(text))# 示例en_text = "I can't believe it!"ch_text = "我不相信这是真的！"print("英文分词：", tokenize_english(en_text))print("中文分词：", tokenize_chinese(ch_text))

from nltk.corpus import stopwords# 下载停用词（若未下载）# nltk.download('stopwords')stop_words_en = set(stopwords.words('english'))stop_words_zh = set(["的", "了", "在", "是", "我", "你", "他", "她", "它"])  # 示例def remove_stopwords(tokens, lang='en'):    if lang == 'en':        return [t for t in tokens if t.lower() not in stop_words_en]    elif lang == 'zh':        return [t for t in tokens if t not in stop_words_zh]    return tokens# 示例tokens = ['I', 'love', 'natural', 'language', 'processing']filtered = remove_stopwords(tokens, 'en')print("去除停用词后：", filtered)# 输出：['love', 'natural', 'language', 'processing']

from nltk.stem import PorterStemmer, WordNetLemmatizerstemmer = PorterStemmer()lemmatizer = WordNetLemmatizer()words = ['running', 'runs', 'better', 'geese']print("词干提取：", [stemmer.stem(w) for w in words])print("词形还原：", [lemmatizer.lemmatize(w, pos='v') for w in words])# 输出：# 词干提取： ['run', 'run', 'better', 'gees']# 词形还原： ['run', 'run', 'better', 'geese']

import reimport jiebaimport spacyfrom nltk.corpus import stopwordsfrom nltk.stem import WordNetLemmatizerclass TextPreprocessor:    """文本预处理流水线"""    def __init__(self, lang='en', use_lemmatization=True):        self.lang = lang        self.use_lemmatization = use_lemmatization        if lang == 'en':            self.nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])            self.stopwords = set(stopwords.words('english'))            if use_lemmatization:                self.lemmatizer = WordNetLemmatizer()        elif lang == 'zh':            # 中文无需spaCy，用jieba            self.stopwords = set(["的", "了", "在", "是", "我", "你", "他", "她", "它", "我们", "你们", "他们"])            # 可加载更大的中文停用词表        else:            raise ValueError("lang must be 'en' or 'zh'")    def clean(self, text):        """清洗"""        # 移除HTML        text = re.sub(r'<[^>]+>', '', text)        # 移除URL        text = re.sub(r'http[s]?://\S+', '', text)        # 移除@提及        text = re.sub(r'@\w+', '', text)        # 保留基本字符        if self.lang == 'en':            text = re.sub(r'[^a-zA-Z\s\.\,\!\?]', '', text)        else:            text = re.sub(r'[^\w\s\u4e00-\u9fff。，！？、]', '', text)        # 合并空格        text = re.sub(r'\s+', ' ', text).strip()        return text    def tokenize(self, text):        """分词"""        if self.lang == 'en':            doc = self.nlp(text)            return [token.text for token in doc]        else:            return list(jieba.cut(text))    def normalize_token(self, token):        """对单个token归一化"""        # 转为小写（英文）        if self.lang == 'en':            token = token.lower()        # 词形还原        if self.lang == 'en' and self.use_lemmatization:            token = self.lemmatizer.lemmatize(token)        return token    def process(self, text):        """完整流水线"""        # 清洗        text = self.clean(text)        # 分词        tokens = self.tokenize(text)        # 去除停用词、归一化        processed = []        for token in tokens:            norm = self.normalize_token(token)            if norm not in self.stopwords and len(norm) > 1:  # 过滤单字符                processed.append(norm)        return processed# 测试英文prep_en = TextPreprocessor(lang='en')text_en = "<p>I absolutely LOVED the movie!!! It was soooo amazing 😍 http://example.com</p>"result_en = prep_en.process(text_en)print("英文预处理结果：", result_en)# 测试中文prep_zh = TextPreprocessor(lang='zh')text_zh = "<p>这家店的奶茶真的超级好喝！！！服务态度也超赞的～～下次还会来💯</p>"result_zh = prep_zh.process(text_zh)print("中文预处理结果：", result_zh)

def preprocess_tweet(tweet):    """针对Twitter的专用预处理"""    # 移除RT（转发标记）    tweet = re.sub(r'^RT ', '', tweet)    # 移除话题标签的#号，但保留文字    tweet = re.sub(r'#(\w+)', r'\1', tweet)    # 统一表情符号    tweet = tweet.replace('😍', ' heart_eyes ').replace('😂', ' laughing ')    # 调用通用预处理    return prep_en.process(tweet)tweets = [    "RT @user: I can't believe how good this movie is!!! #awesome",    "Worst service ever 😤😤😤 @company",    "Just bought the new iPhone. Loving it! 😍😍😍"]for tweet in tweets:    processed = preprocess_tweet(tweet)    print(f"原始: {tweet}")    print(f"处理后: {processed}\n")

原始: RT @user: I can't believe how good this movie is!!! #awesome处理后: ['believe', 'good', 'movie', 'awesome']原始: Worst service ever 😤😤😤 @company处理后: ['worst', 'service', 'ever']原始: Just bought the new iPhone. Loving it! 😍😍😍处理后: ['bought', 'new', 'iphone', 'loving', 'heart_eyes']

"I absolutely LOVED the movie!!! It was soooo amazing 😍😍😍 #mustwatch"

['absolutely', 'love', 'movie', 'amazing', 'mustwatch']