从零基础，到 NLP 应用精通

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

docs = [
    "春天的西湖游人如织，苏堤春晓桃红柳绿，断桥边杨柳依依。",
    "夜晚的西湖别有韵味，湖面倒映着灯火，断桥与雷峰塔遥遥相望。",
    "梯度下降是训练神经网络最常用的优化算法，沿着损失下降的方向更新参数。",
    "反向传播利用链式法则逐层计算梯度，神经网络通过不断更新参数来降低损失。",
    "杭帮菜讲究清淡鲜嫩，西湖醋鱼与龙井虾仁是最有名的两道菜。",
]

# 中文没有空格分词，改用"字符级"分析器（按单字切分，短文本效果最好）
vec = TfidfVectorizer(analyzer="char")
matrix = vec.fit_transform(docs)
sim = cosine_similarity(matrix)

for i in range(len(docs)):
    for j in range(i + 1, len(docs)):
        print(f"文档{i+1} vs 文档{j+1}: {sim[i][j]:.3f}")
# pip install scikit-learn 后即可运行

import statistics
from collections import Counter

def chars(text):                      # 只留汉字，丢掉标点
    return [c for c in text if "\u4e00" <= c <= "\u9fff"]

corpus = {                            # 每位作者至少 2 段样本
    "作者A": ["……样本1……", "……样本2……"],
    "作者B": ["……样本1……", "……样本2……"],
}
unknown = "……待判定文本……"
top_n = 30                            # 取最高频的前 30 个字

# 1. 每个文本统计"字频比例"
freqs = {}
for samples in corpus.values():
    for t in samples:
        cs = chars(t); n = len(cs) or 1
        freqs[t] = Counter({c: v / n for c, v in Counter(cs).items()})
cs = chars(unknown); n = len(cs) or 1
freqs["待判定"] = Counter({c: v / n for c, v in Counter(cs).items()})

# 2. 取全语料最高频的 top_n 个字
total = Counter()
for c in freqs.values():
    total.update(c)
top = [w for w, _ in total.most_common(top_n)]

# 3. 对每个字做 z 分数（跨文本标准化）
z = {}
for t, f in freqs.items():
    vals = [f[w] for w in top]
    mean = statistics.mean(vals)
    sd = statistics.pstdev(vals) or 1e-9
    z[t] = {w: (v - mean) / sd for w, v in zip(top, vals)}

# 4. 每位作者取样本平均 z 分数，算 Delta 距离
for author, samples in corpus.items():
    profile = {w: statistics.mean(z[t][w] for t in samples) for w in top}
    delta = statistics.mean(abs(z["待判定"][w] - profile[w]) for w in top)
    print(f"{author}: Delta = {delta:.3f}")

# Delta 越小，待判定文本与该作者的文风越接近。

import re
import jieba
from collections import Counter

with open("corpus.txt", encoding="utf-8") as f:
    text = f.read()

# jieba 是最常用的中文分词库: pip install jieba
words = [w for w in jieba.lcut(text) if re.search(r"[\u4e00-\u9fff]", w)]
counter = Counter(words)

print("总字数(含标点):", len(text))
print("总词数:", len(words))
print("类符数:", len(counter))
print("TTR 类符/形符比:", round(len(counter) / len(words), 3))
print("\n前 20 个高频词:")
for w, n in counter.most_common(20):
    print(f"{w}\t{n}")

# 字符级 bigram 语言模型：语言模型的一切，从"数一数"开始
# 纯标准库，pip install 什么都不需要
from collections import Counter, defaultdict
import random

with open("corpus.txt", encoding="utf-8") as f:
    text = f.read()
chars = [c for c in text if "\u4e00" <= c <= "\u9fff" or c == "。"]

# 1. 训练：数"上一个字 → 下一个字"的出现次数
model = defaultdict(Counter)
for a, b in zip(chars, chars[1:]):
    model[a][b] += 1

# 2. 预测：给定一个字，输出它后面最常跟的字（P(下字|上字)）
prev = "语"
counts = model[prev]
total = sum(counts.values())
print(f"「{prev}」后面最可能跟：")
for ch, n in counts.most_common(5):
    print(f"  {ch}   P = {n / total:.2f}")

# 3. 续写：按概率随机采样，一路走 20 个字
out = [prev]
for _ in range(20):
    chs, ns = zip(*model[out[-1]].items()) if model[out[-1]] else (["。"], [1])
    out.append(random.choices(chs, ns)[0])
print("模型续写：", "".join(out))

# Cohen's kappa：两位评分者的一致程度（校正"碰巧一致"）
# 打分制随意，这里用 0差 / 1中 / 2良 / 3优；纯标准库
from collections import Counter

rater_a = [3, 2, 3, 1, 0, 2, 3, 1, 2, 3]   # 评分者A（比如你）
rater_b = [3, 1, 2, 1, 0, 2, 3, 0, 2, 2]   # 评分者B（比如同学或 LLM）

n = len(rater_a)
po = sum(a == b for a, b in zip(rater_a, rater_b)) / n          # 实际一致率
ca, cb = Counter(rater_a), Counter(rater_b)
cats = set(rater_a) | set(rater_b)
pe = sum(ca[c] * cb[c] for c in cats) / n ** 2                  # 碰巧一致率
kappa = (po - pe) / (1 - pe) if pe < 1 else 1.0

print(f"实际一致率 po = {po:.2f}")
print(f"碰巧一致率 pe = {pe:.2f}")
print(f"Cohen's kappa = {kappa:.2f}   （0.41–0.60 中等，0.61–0.80 强）")