自然语言处理（一）：NLP 入门与文本预处理

1
2
3
4
5
6
7

原始文本
  -> 清洗      （去除 HTML 标签、URL、邮箱地址和无意义字符）
  -> 分词      （切分为单词或子词单元）
  -> 规范化    （转为小写、还原词形，必要时提取词干）
  -> 停用词过滤（根据需要移除 "the"、"is"、"at" 等高频无意义词）
  -> 向量化    （使用 BoW、TF-IDF 或词嵌入表示）
  -> 模型

1
2

pip install nltk spacy scikit-learn matplotlib numpy pandas beautifulsoup4
python -m spacy download en_core_web_sm

1
2
3

import nltk
for pkg in ['punkt', 'stopwords', 'wordnet', 'averaged_perceptron_tagger']:
    nltk.download(pkg, quiet=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

import re

def clean_text(text: str) -> str:
    """去除 HTML 标签、URL、邮箱地址和非字母字符，统一空白符。"""
    text = re.sub(r'<[^>]+>', '', text)              # 去掉 HTML 标签
    text = re.sub(r'http\S+|www\.\S+', '', text)     # 去掉 URL
    text = re.sub(r'\S+@\S+', '', text)              # 去掉邮箱
    text = re.sub(r'[^a-zA-Z\s]', '', text)          # 只保留字母和空格
    text = re.sub(r'\s+', ' ', text).strip()         # 统一空白符
    return text

raw = """<p>Check out https://example.com for info!</p>
Contact info@test.com. Price: $29.99"""
print(clean_text(raw))
# Check out for info Contact Price

1
2
3

HTML_RE = re.compile(r'<[^>]+>')
URL_RE  = re.compile(r'http\S+|www\.\S+')
text = URL_RE.sub('', HTML_RE.sub('', text))

1
2

from bs4 import BeautifulSoup
text = BeautifulSoup(html_text, 'html.parser').get_text(' ', strip=True)

1
2
3
4
5
6
7
8

# 简单粗暴：遇到缩写和标点就出问题
"Don't split can't".split()
# ["Don't", 'split', "can't"]

from nltk.tokenize import word_tokenize
tokens = word_tokenize("Dr. Smith earned $150,000 in 2023! Isn't that amazing?")
# ['Dr.', 'Smith', 'earned', '$', '150,000', 'in', '2023', '!',
#  'Is', "n't", 'that', 'amazing', '?']

1
2
3
4

from nltk.tokenize import sent_tokenize
text = "Dr. Johnson works at A.I. Corp. He earned his Ph.D. in 2010."
sent_tokenize(text)
# ['Dr. Johnson works at A.I. Corp.', 'He earned his Ph.D. in 2010.']

1
2
3
4
5
6
7

语料: "low" x5, "lower" x2, "newest" x6, "widest" x3
初始:  l o w  /  l o w e r  /  n e w e s t  /  w i d e s t

合并 1: (e, s) -> es        # 在 "newest" 和 "widest" 中频繁出现
合并 2: (es, t) -> est
合并 3: (l, o) -> lo
...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

from collections import defaultdict

def get_stats(vocab):
    """统计相邻符号对的频率"""
    pairs = defaultdict(int)
    for word, freq in vocab.items():
        symbols = word.split()
        for i in range(len(symbols) - 1):
            pairs[symbols[i], symbols[i + 1]] += freq
    return pairs

def merge_vocab(pair, vocab):
    bigram = ' '.join(pair)
    replacement = ''.join(pair)
    return {w.replace(bigram, replacement): f for w, f in vocab.items()}

vocab = {'l o w </w>': 5, 'l o w e r </w>': 2,
         'n e w e s t </w>': 6, 'w i d e s t </w>': 3}

for step in range(5):
    pairs = get_stats(vocab)
    if not pairs:
        break
    best = max(pairs, key=pairs.get)
    vocab = merge_vocab(best, vocab)
    print(f"合并 {step + 1}: {best} -> {''.join(best)}")

1
2

"Apple Inc. sells apples in APPLE stores".lower()
# "apple inc. sells apples in apple stores"

1
2
3
4
5

from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
for w in ['running', 'easily', 'connection']:
    print(f"{w} -> {stemmer.stem(w)}")
# running -> run, easily -> easili, connection -> connect

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

import spacy
nlp = spacy.load('en_core_web_sm')
doc = nlp("The geese were running and swimming better than the mice")
for token in doc:
    print(f"{token.text:10} -> {token.lemma_:10} ({token.pos_})")
# geese      -> goose      (NOUN)
# were       -> be         (AUX)
# running    -> run        (VERB)
# swimming   -> swim       (VERB)
# better     -> well       (ADV)
# mice       -> mouse      (NOUN)

1
2
3
4
5
6
7

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

stop_words = set(stopwords.words('english'))
text = "The quick brown fox jumps over the lazy dog"
filtered = [w for w in word_tokenize(text.lower()) if w not in stop_words]
# ['quick', 'brown', 'fox', 'jumps', 'lazy', 'dog']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd

docs = [
    "I love machine learning",
    "Machine learning is amazing",
    "I love deep learning and machine learning",
]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(docs)
print(pd.DataFrame(X.toarray(), columns=vectorizer.get_feature_names_out()))

1
2
3
4

   amazing  and  deep  is  learning  love  machine
0        0    0     0   0         1     1        1
1        1    0     0   1         1     0        1
2        0    1     1   0         2     1        1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

docs = [
    "Machine learning is a subset of artificial intelligence",
    "Deep learning is a subset of machine learning",
    "Natural language processing uses machine learning",
    "Computer vision uses deep learning techniques",
]

tfidf = TfidfVectorizer()
X = tfidf.fit_transform(docs)
df = pd.DataFrame(X.toarray(), columns=tfidf.get_feature_names_out())

for i, doc in enumerate(docs):
    top = df.iloc[i].sort_values(ascending=False).head(3)
    print(f"文档 {i + 1}: {dict(top.round(3))}")

1
2
3
4
5
6
7
8

tfidf = TfidfVectorizer(
    max_features=5_000,    # 限制词表大小
    min_df=2,              # 去掉只出现一次的词
    max_df=0.8,            # 去掉出现在 80% 以上文档中的词（相当于停用词）
    ngram_range=(1, 2),    # 一元 + 二元，捕捉短语
    sublinear_tf=True,     # 对 TF 取对数，抑制重复
    stop_words='english',
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

import re
import spacy
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize

class TextPreprocessor:
    """灵活配置的英文文本预处理流水线。"""

    def __init__(self, use_lemmatization: bool = True,
                 remove_stopwords: bool = True):
        self.use_lemmatization = use_lemmatization
        self.remove_stopwords = remove_stopwords

        if use_lemmatization:
            # 加载 spaCy 的小型英语模型，禁用解析器和命名实体识别以提高效率
            self.nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])
        else:
            # 使用 NLTK 的 Porter 词干提取器
            self.stemmer = PorterStemmer()

        if remove_stopwords:
            # 获取英文停用词集合
            self.stop_words = set(stopwords.words('english'))

    def clean(self, text: str) -> str:
        # 转为小写
        text = text.lower()
        # 移除 HTML 标签
        text = re.sub(r'<[^>]+>', '', text)
        # 移除 URL 链接
        text = re.sub(r'http\S+|www\.\S+', '', text)
        # 移除电子邮件地址
        text = re.sub(r'\S+@\S+', '', text)
        # 移除非字母字符
        text = re.sub(r'[^a-zA-Z\s]', '', text)
        # 合并多余空格并去除首尾空白
        return re.sub(r'\s+', ' ', text).strip()

    def tokenize_and_normalize(self, text: str) -> list[str]:
        if self.use_lemmatization:
            # 使用 spaCy 进行词形还原
            doc = self.nlp(text)
            tokens = [t.lemma_ for t in doc if not t.is_space]
        else:
            # 使用 NLTK 进行词干提取
            tokens = [self.stemmer.stem(t) for t in word_tokenize(text)]

        if self.remove_stopwords:
            # 移除停用词
            tokens = [t for t in tokens if t not in self.stop_words]
        return tokens

    def preprocess(self, text: str) -> str:
        # 清洗、分词并归一化后重新拼接为字符串
        return ' '.join(self.tokenize_and_normalize(self.clean(text)))

    def preprocess_corpus(self, texts: list[str]) -> list[str]:
        # 对文本集合中的每条文本进行预处理
        return [self.preprocess(t) for t in texts]

# 初始化预处理器，启用词形还原并移除停用词
pre = TextPreprocessor(use_lemmatization=True, remove_stopwords=True)

# 示例文本集合
texts = [
    "Natural Language Processing (NLP) is amazing! Visit https://example.com",
    "Machine learning models are trained on large datasets.",
    "Deep learning has revolutionized computer vision and NLP.",
]

# 打印原始文本与处理后的结果对比
for orig, proc in zip(texts, pre.preprocess_corpus(texts)):
    print(f"原文：  {orig}")
    print(f"处理后：{proc}\n")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import numpy as np

texts = [
    "Congratulations! You've won a $1000 gift card. Call now!",
    "Hey, are we still meeting for dinner tonight?",
    "URGENT: Your account will be closed. Click here immediately!",
    "Can you send me the project report by EOD?",
    "Get rich quick! Amazing investment opportunity!",
    "Don't forget to pick up milk on your way home",
    "You have been selected for a free cruise. Reply YES",
    "Meeting moved to 3pm tomorrow in conference room B",
    "Lose 20 pounds in 2 weeks with this miracle pill!",
    "Thanks for your help with the presentation yesterday",
]
labels = np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0])  # 1=垃圾邮件, 0=正常邮件

# 保留停用词——像 "free"、"now"、"you" 这样的词往往是垃圾邮件的重要特征
pre = TextPreprocessor(use_lemmatization=True, remove_stopwords=False)
processed = pre.preprocess_corpus(texts)

vectorizer = TfidfVectorizer(max_features=50, ngram_range=(1, 2))
X = vectorizer.fit_transform(processed)

X_train, X_test, y_train, y_test = train_test_split(
    X, labels, test_size=0.3, random_state=42, stratify=labels)

model = LogisticRegression(max_iter=1000).fit(X_train, y_train)

new_msgs = ["Can you review my code?", "FREE MONEY!!! Click now!!!"]
new_vecs = vectorizer.transform(pre.preprocess_corpus(new_msgs))
for msg, pred in zip(new_msgs, model.predict(new_vecs)):
    print(f"[{'垃圾邮件' if pred else '正常邮件'}] {msg}")