# sentences = [['the', 'cat', 'sat']]
# unigram = {'the': 0.5, 'cat': 0.4, 'sat': 0.1}
sentences = [
['I', 'have', 'a', 'pen'],
['He', 'has', 'a', 'book'],
['She', 'has', 'a', 'cat']
]
# 定义语言模型
unigram = {
'I': 1/12,
'have': 1/12,
'a': 3/12,
'pen': 1/12,
'He': 1/12,
'has': 2/12,
'book': 1/12,
'She': 1/12,
'cat': 1/12
}
# 初始化联合概率和N数量
uni_prob = 1.0
N = 0
# 外层遍历
for sentence in sentences:
N += len(sentence)
# 内层遍历
for word in sentence:
# 得到每个单词的概率
prob = unigram[word]
# 计算联合概率
uni_prob *= prob
# 计算ppl
ppl = uni_prob ** (- 1.0/N)
print("ppl-->", ppl)