一天一个Python知识点——Day 162:spaCy基础

pip install spacy

# 英文小模型（推荐入门）python -m spacy download en_core_web_sm# 英文中模型python -m spacy download en_core_web_md# 英文大模型（包含词向量）python -m spacy download en_core_web_lg# 中文模型python -m spacy download zh_core_web_sm

import spacynlp = spacy.load("en_core_web_sm")doc = nlp("Hello, spaCy!")print([token.text for token in doc])# 输出：['Hello', ',', 'spaCy', '!']

import spacynlp = spacy.load("en_core_web_sm")text = "Apple is looking at buying U.K. startup for $1 billion"doc = nlp(text)# 遍历Tokenfor token in doc:    print(f"{token.text:10}{token.pos_:8}{token.dep_:10}{token.head.text}")# 遍历句子for sent in doc.sents:    print(sent.text)# 获取命名实体for ent in doc.ents:    print(f"{ent.text:15}{ent.label_:10}{ent.start_char}-{ent.end_char}")

doc = nlp("Let's go to N.Y.C. for pizza.")tokens = [token.text for token in doc]print(tokens)# 输出：['Let', "'s", 'go', 'to', 'N.Y.C.', 'for', 'pizza', '.']

doc = nlp("I love natural language processing")for token in doc:    print(f"{token.text:10}{token.pos_:8}{token.tag_:6}")

I          PRON     PRPlove       VERB     VBPnatural    ADJ      JJlanguage   NOUN     NNprocessing NOUN     NN

doc = nlp("The quick brown fox jumps over the lazy dog.")for token in doc:    print(f"{token.text:10}{token.dep_:12}{token.head.text:10}")

fox        nsubj         jumpsjumps      ROOT          jumpsover       prep          jumpsdog        pobj          over

from spacy import displacydisplacy.render(doc, style="dep", jupyter=True)  # 在Notebook中# 或保存为SVGsvg = displacy.render(doc, style="dep")with open("dependency.svg", "w") as f:    f.write(svg)

doc = nlp("Apple Inc. was founded by Steve Jobs in Cupertino.")for ent in doc.ents:    print(f"{ent.text:15}{ent.label_:10}{spacy.explain(ent.label_)}")

Apple Inc.      ORG        Companies, agencies, institutionsSteve Jobs      PERSON     People, including fictionalCupertino       GPE        Countries, cities, states

import spacynlp = spacy.load("en_core_web_md")# 词向量word1 = nlp("king")word2 = nlp("queen")print(f"相似度：{word1.similarity(word2):.3f}")# 句子向量sent1 = nlp("I like cats")sent2 = nlp("I love dogs")print(f"句子相似度：{sent1.similarity(sent2):.3f}")

from spacy.matcher import Matchernlp = spacy.load("en_core_web_sm")matcher = Matcher(nlp.vocab)# 定义模式：匹配"make"后跟一个名词pattern = [{"LEMMA": "make"}, {"POS": "NOUN"}]matcher.add("MAKE_NOUN", [pattern])doc = nlp("She made a cake. They make pizza.")matches = matcher(doc)for match_id, start, end in matches:    span = doc[start:end]    print(span.text)# 输出：made a cake, make pizza

from spacy.matcher import PhraseMatchermatcher = PhraseMatcher(nlp.vocab)patterns = [nlp("New York"), nlp("San Francisco")]matcher.add("CITIES", patterns)doc = nlp("I love New York and San Francisco")matches = matcher(doc)for match_id, start, end in matches:    print(doc[start:end].text)

def my_component(doc):    print("自定义组件被调用")    # 对doc进行自定义处理    return docnlp.add_pipe(my_component, last=True)doc = nlp("Hello world")  # 会打印"自定义组件被调用"

import spacyfrom collections import defaultdictnlp = spacy.load("en_core_web_sm")def extract_relations(text):    doc = nlp(text)    relations = []    for sent in doc.sents:        persons = [ent for ent in sent.ents if ent.label_ == "PERSON"]        orgs = [ent for ent in sent.ents if ent.label_ == "ORG"]        if not persons or not orgs:            continue        # 寻找连接人物和组织的动词        verbs = [token for token in sent if token.pos_ == "VERB"]        for person in persons:            for org in orgs:                for verb in verbs:                    # 检查人物和组织是否与动词有依存关系                    # 简化：人物可能是nsubj，组织可能是dobj或pobj                    if verb.dep_ in ["ROOT", "conj"]:                        subj = [w for w in verb.children if w.dep_ in ["nsubj", "agent"] and w in person]                        obj = [w for w in verb.children if w.dep_ in ["dobj", "pobj"] and w in org]                        if subj and obj:                            relations.append((person.text, verb.lemma_, org.text))                        # 也有可能反过来                        subj = [w for w in verb.children if w.dep_ in ["nsubj", "agent"] and w in org]                        obj = [w for w in verb.children if w.dep_ in ["dobj", "pobj"] and w in person]                        if subj and obj:                            relations.append((org.text, verb.lemma_, person.text))    return relations# 测试news = """Apple Inc. announced that Tim Cook will open a new store in London.Elon Musk's Tesla plans to build a factory in Berlin."""relations = extract_relations(news)for subj, verb, obj in relations:    print(f"{subj}{verb}{obj}")

Tim Cook open Apple Inc.Elon Musk build Tesla

# 训练NER的示例import spacyfrom spacy.training import Examplenlp = spacy.blank("en")  # 创建空白模型ner = nlp.add_pipe("ner")# 添加标签ner.add_label("PRODUCT")# 准备训练数据TRAIN_DATA = [    ("Apple released the iPhone", {"entities": [(0, 5, "ORG"), (15, 21, "PRODUCT")]}),    ("Samsung Galaxy is popular", {"entities": [(0, 14, "PRODUCT")]})]# 开始训练optimizer = nlp.begin_training()for epoch in range(10):    for text, annotations in TRAIN_DATA:        doc = nlp.make_doc(text)        example = Example.from_dict(doc, annotations)        nlp.update([example], sgd=optimizer)