Average Normalized Levenshtein Similarity(ANLS)用编辑距离衡量预测文字与答案的接近程度。字串越长,可容忍的字元差异也越多;因此先将 Levenshtein 距离除以两个字串中较长者的长度,再把结果转成相似度。
ST-VQA 原始论文使用 0.5 门槛:标准化距离超过 0.5 的答案记为 0 分,其余使用 1 - 距離,最后才对多笔样本取平均。只计算 1 - 距離 而不套门槛,得到的是平均 NLS,分数可能与 ANLS 不同。下方范例示范逐对字串计分;若资料集一题有多个可接受答案,还需依该资料集的评分规则选取答案。
参考资料
导入必要的库
首先,需要导入一些必要的库,特别是由 torchmetrics 实作的 EditDistance:
from typing import Any, Literal, Optional, Sequence, Union
import torch
from torch import Tensor
from torchmetrics.metric import Metric
from torchmetrics.text import EditDistance
from torchmetrics.utilities.data import dim_zero_cat
EditDistance 回传逐对字串的编辑距离;接著依较长字串的长度正规化,再套用 ANLS 的 0.5 门槛。
实作标准化功能
这个类别继承 torchmetrics.metric.Metric,因此要实作 update 和 compute:
class NormalizedLevenshteinSimilarity(Metric):
def __init__(
self,
substitution_cost: int = 1,
reduction: Optional[Literal["mean", "sum", "none"]] = "mean",
**kwargs: Any
) -> None:
super().__init__(**kwargs)
self.edit_distance = EditDistance(
substitution_cost=substitution_cost,
reduction=None # Set to None to get distances for all string pairs
)
# ...
这里有几个要点:
- 确保输入的
preds和target是字串列表,否则函数就会计算到「字元」的部分。 - 计算每个字串的最大长度,这样才能进行标准化。
def update(self, preds: Union[str, Sequence[str]], target: Union[str, Sequence[str]]) -> None:
"""Update state with predictions and targets."""
if isinstance(preds, str):
preds = [preds]
if isinstance(target, str):
target = [target]
if len(preds) != len(target):
raise ValueError("preds and target must have the same length")
distances = self.edit_distance(preds, target)
max_lengths = torch.tensor([
max(len(p), len(t))
for p, t in zip(preds, target)
], dtype=torch.float, device=distances.device)
ratio = torch.where(
max_lengths == 0,
torch.zeros_like(distances).float(),
distances.float() / max_lengths
)
nls_values = torch.where(
ratio <= 0.5,
1 - ratio,
torch.zeros_like(ratio),
)
# ...
实作 reduction 参数
reduction="mean" 会平均每对字串套用门槛后的分数。
sum 回传总和;none 回传每对字串的分数。
def _compute(
self,
nls_score: Tensor,
num_elements: Union[Tensor, int],
) -> Tensor:
"""Compute the ANLS over state."""
if nls_score.numel() == 0 or num_elements == 0:
return torch.tensor(0, dtype=torch.int32)
if self.reduction == "mean":
return nls_score.sum() / num_elements
if self.reduction == "sum":
return nls_score.sum()
if self.reduction is None or self.reduction == "none":
return nls_score
def compute(self) -> torch.Tensor:
"""Compute the NLS over state."""
if self.reduction == "none" or self.reduction is None:
return self._compute(dim_zero_cat(self.nls_values_list), 1)
return self._compute(self.nls_score, self.num_elements)
reduction="none" 会回传每对字串套用门槛后的分数;dim_zero_cat 将储存的分数合并为一个 Tensor。
程式码
完整的实作如下:
from typing import Any, Literal, Optional, Sequence, Union
import torch
from torch import Tensor
from torchmetrics.metric import Metric
from torchmetrics.text import EditDistance
from torchmetrics.utilities.data import dim_zero_cat
class NormalizedLevenshteinSimilarity(Metric):
"""
Normalized Levenshtein Similarity (NLS) is a metric that computes the
normalized Levenshtein similarity between two sequences.
This metric is 1 - normalized edit distance when the distance is at most 0.5,
where `levenshtein_distance` is the Levenshtein distance between the two
sequences and `max_length` is the maximum length of the two sequences.
NLS aims to provide a similarity measure for character sequences
(such as text), making it useful in areas like text similarity analysis,
Optical Character Recognition (OCR), and Natural Language Processing (NLP).
This class inherits from `Metric` and uses the `EditDistance` class to
compute the Levenshtein distance.
Inputs to the ``update`` and ``compute`` methods are as follows:
- ``preds`` (:class:`~Union[str, Sequence[str]]`):
Predicted text sequences or a collection of sequences.
- ``target`` (:class:`~Union[str, Sequence[str]]`):
Target text sequences or a collection of sequences.
Output from the ``compute`` method is as follows:
- ``nls`` (:class:`~torch.Tensor`): A tensor containing the NLS value.
Returns 0.0 when there are no samples; otherwise, it returns the NLS.
Args:
substitution_cost:
The cost of substituting one character for another. Default is 1.
reduction:
Method to aggregate metric scores.
Default is 'mean', options are 'sum' or None.
- ``'mean'``: takes the mean over samples, which is ANLS.
- ``'sum'``: takes the sum over samples
- ``None`` or ``'none'``: returns the score per sample
kwargs: Additional keyword arguments.
Example::
Multiple strings example:
>>> metric = NormalizedLevenshteinSimilarity(reduction=None)
>>> preds = ["rain", "lnaguaeg"]
>>> target = ["shine", "language"]
>>> metric(preds, target)
tensor([0.0000, 0.5000])
>>> metric = NormalizedLevenshteinSimilarity(reduction="mean")
>>> metric(preds, target)
tensor(0.2500)
"""
def __init__(
self,
substitution_cost: int = 1,
reduction: Optional[Literal["mean", "sum", "none"]] = "mean",
**kwargs: Any
) -> None:
super().__init__(**kwargs)
self.edit_distance = EditDistance(
substitution_cost=substitution_cost,
reduction=None # Set to None to get distances for all string pairs
)
allowed_reduction = (None, "mean", "sum", "none")
if reduction not in allowed_reduction:
raise ValueError(
f"Expected argument `reduction` to be one of {allowed_reduction}, but got {reduction}")
self.reduction = reduction
if self.reduction == "none" or self.reduction is None:
self.add_state(
"nls_values_list",
default=[],
dist_reduce_fx="cat"
)
else:
self.add_state(
"nls_score",
default=torch.tensor(0.0),
dist_reduce_fx="sum"
)
self.add_state(
"num_elements",
default=torch.tensor(0),
dist_reduce_fx="sum"
)
def update(self, preds: Union[str, Sequence[str]], target: Union[str, Sequence[str]]) -> None:
"""Update state with predictions and targets."""
if isinstance(preds, str):
preds = [preds]
if isinstance(target, str):
target = [target]
if len(preds) != len(target):
raise ValueError("preds and target must have the same length")
distances = self.edit_distance(preds, target)
max_lengths = torch.tensor([
max(len(p), len(t))
for p, t in zip(preds, target)
], dtype=torch.float, device=distances.device)
ratio = torch.where(
max_lengths == 0,
torch.zeros_like(distances).float(),
distances.float() / max_lengths
)
nls_values = torch.where(
ratio <= 0.5,
1 - ratio,
torch.zeros_like(ratio),
)
if self.reduction == "none" or self.reduction is None:
self.nls_values_list.append(nls_values)
else:
self.nls_score += nls_values.sum()
self.num_elements += nls_values.shape[0]
def _compute(
self,
nls_score: Tensor,
num_elements: Union[Tensor, int],
) -> Tensor:
"""Compute the ANLS over state."""
if nls_score.numel() == 0 or num_elements == 0:
return torch.tensor(0, dtype=torch.int32)
if self.reduction == "mean":
return nls_score.sum() / num_elements
if self.reduction == "sum":
return nls_score.sum()
if self.reduction is None or self.reduction == "none":
return nls_score
def compute(self) -> torch.Tensor:
"""Compute the NLS over state."""
if self.reduction == "none" or self.reduction is None:
return self._compute(dim_zero_cat(self.nls_values_list), 1)
return self._compute(self.nls_score, self.num_elements)
if __name__ == "__main__":
anls = NormalizedLevenshteinSimilarity(reduction='mean')
preds = ["rain", "lnaguaeg"]
target = ["shine", "language"]
print(anls(preds, target))
最后
这份程式示范单一答案、逐对字串的 ANLS 计算。要与特定资料集的官方评分一致,还需核对大小写与空白正规化、多答案选取方式,以及空字串的处理规则。
