返回博客
一般文章

实现 ANLS

用 torchmetrics 计算归一化编辑距离,补上 ST-VQA 评分使用的 0.5 阈值。

Average Normalized Levenshtein Similarity(ANLS)用编辑距离衡量预测文字与答案的接近程度。字串越长,可容忍的字元差异也越多;因此先将 Levenshtein 距离除以两个字串中较长者的长度,再把结果转成相似度。

ST-VQA 原始论文使用 0.5 门槛:标准化距离超过 0.5 的答案记为 0 分,其余使用 1 - 距離,最后才对多笔样本取平均。只计算 1 - 距離 而不套门槛,得到的是平均 NLS,分数可能与 ANLS 不同。下方范例示范逐对字串计分;若资料集一题有多个可接受答案,还需依该资料集的评分规则选取答案。

参考资料

导入必要的库

首先,需要导入一些必要的库,特别是由 torchmetrics 实作的 EditDistance:

python
from typing import Any, Literal, Optional, Sequence, Union

import torch
from torch import Tensor
from torchmetrics.metric import Metric
from torchmetrics.text import EditDistance
from torchmetrics.utilities.data import dim_zero_cat

EditDistance 回传逐对字串的编辑距离;接著依较长字串的长度正规化,再套用 ANLS 的 0.5 门槛。

实作标准化功能

这个类别继承 torchmetrics.metric.Metric,因此要实作 update 和 compute:

python
class NormalizedLevenshteinSimilarity(Metric):

    def __init__(
        self,
        substitution_cost: int = 1,
        reduction: Optional[Literal["mean", "sum", "none"]] = "mean",
        **kwargs: Any
    ) -> None:
        super().__init__(**kwargs)
        self.edit_distance = EditDistance(
            substitution_cost=substitution_cost,
            reduction=None  # Set to None to get distances for all string pairs
        )

        # ...

这里有几个要点:

  1. 确保输入的 preds 和 target 是字串列表,否则函数就会计算到「字元」的部分。
  2. 计算每个字串的最大长度,这样才能进行标准化。
python
def update(self, preds: Union[str, Sequence[str]], target: Union[str, Sequence[str]]) -> None:
    """Update state with predictions and targets."""

    if isinstance(preds, str):
        preds = [preds]
    if isinstance(target, str):
        target = [target]
    if len(preds) != len(target):
        raise ValueError("preds and target must have the same length")

    distances = self.edit_distance(preds, target)
    max_lengths = torch.tensor([
        max(len(p), len(t))
        for p, t in zip(preds, target)
    ], dtype=torch.float, device=distances.device)

    ratio = torch.where(
        max_lengths == 0,
        torch.zeros_like(distances).float(),
        distances.float() / max_lengths
    )

    nls_values = torch.where(
        ratio <= 0.5,
        1 - ratio,
        torch.zeros_like(ratio),
    )

    # ...

实作 reduction 参数

reduction="mean" 会平均每对字串套用门槛后的分数。

sum 回传总和;none 回传每对字串的分数。

python
def _compute(
    self,
    nls_score: Tensor,
    num_elements: Union[Tensor, int],
) -> Tensor:
    """Compute the ANLS over state."""
    if nls_score.numel() == 0 or num_elements == 0:
        return torch.tensor(0, dtype=torch.int32)
    if self.reduction == "mean":
        return nls_score.sum() / num_elements
    if self.reduction == "sum":
        return nls_score.sum()
    if self.reduction is None or self.reduction == "none":
        return nls_score

def compute(self) -> torch.Tensor:
    """Compute the NLS over state."""
    if self.reduction == "none" or self.reduction is None:
        return self._compute(dim_zero_cat(self.nls_values_list), 1)
    return self._compute(self.nls_score, self.num_elements)

reduction="none" 会回传每对字串套用门槛后的分数;dim_zero_cat 将储存的分数合并为一个 Tensor。

程式码

完整的实作如下:

python
from typing import Any, Literal, Optional, Sequence, Union

import torch
from torch import Tensor
from torchmetrics.metric import Metric
from torchmetrics.text import EditDistance
from torchmetrics.utilities.data import dim_zero_cat

class NormalizedLevenshteinSimilarity(Metric):
    """
    Normalized Levenshtein Similarity (NLS) is a metric that computes the
    normalized Levenshtein similarity between two sequences.
    This metric is 1 - normalized edit distance when the distance is at most 0.5,
    where `levenshtein_distance` is the Levenshtein distance between the two
    sequences and `max_length` is the maximum length of the two sequences.

    NLS aims to provide a similarity measure for character sequences
    (such as text), making it useful in areas like text similarity analysis,
    Optical Character Recognition (OCR), and Natural Language Processing (NLP).

    This class inherits from `Metric` and uses the `EditDistance` class to
    compute the Levenshtein distance.

    Inputs to the ``update`` and ``compute`` methods are as follows:

    - ``preds`` (:class:`~Union[str, Sequence[str]]`):
        Predicted text sequences or a collection of sequences.
    - ``target`` (:class:`~Union[str, Sequence[str]]`):
        Target text sequences or a collection of sequences.

    Output from the ``compute`` method is as follows:

    - ``nls`` (:class:`~torch.Tensor`): A tensor containing the NLS value.
        Returns 0.0 when there are no samples; otherwise, it returns the NLS.

    Args:
        substitution_cost:
            The cost of substituting one character for another. Default is 1.
        reduction:
            Method to aggregate metric scores.
            Default is 'mean', options are 'sum' or None.

            - ``'mean'``: takes the mean over samples, which is ANLS.
            - ``'sum'``: takes the sum over samples
            - ``None`` or ``'none'``: returns the score per sample

        kwargs: Additional keyword arguments.

    Example::
        Multiple strings example:

        >>> metric = NormalizedLevenshteinSimilarity(reduction=None)
        >>> preds = ["rain", "lnaguaeg"]
        >>> target = ["shine", "language"]
        >>> metric(preds, target)
        tensor([0.0000, 0.5000])
        >>> metric = NormalizedLevenshteinSimilarity(reduction="mean")
        >>> metric(preds, target)
        tensor(0.2500)
    """

    def __init__(
        self,
        substitution_cost: int = 1,
        reduction: Optional[Literal["mean", "sum", "none"]] = "mean",
        **kwargs: Any
    ) -> None:
        super().__init__(**kwargs)
        self.edit_distance = EditDistance(
            substitution_cost=substitution_cost,
            reduction=None  # Set to None to get distances for all string pairs
        )

        allowed_reduction = (None, "mean", "sum", "none")
        if reduction not in allowed_reduction:
            raise ValueError(
                f"Expected argument `reduction` to be one of {allowed_reduction}, but got {reduction}")
        self.reduction = reduction

        if self.reduction == "none" or self.reduction is None:
            self.add_state(
                "nls_values_list",
                default=[],
                dist_reduce_fx="cat"
            )
        else:
            self.add_state(
                "nls_score",
                default=torch.tensor(0.0),
                dist_reduce_fx="sum"
            )
            self.add_state(
                "num_elements",
                default=torch.tensor(0),
                dist_reduce_fx="sum"
            )

    def update(self, preds: Union[str, Sequence[str]], target: Union[str, Sequence[str]]) -> None:
        """Update state with predictions and targets."""
        if isinstance(preds, str):
            preds = [preds]
        if isinstance(target, str):
            target = [target]
        if len(preds) != len(target):
            raise ValueError("preds and target must have the same length")

        distances = self.edit_distance(preds, target)
        max_lengths = torch.tensor([
            max(len(p), len(t))
            for p, t in zip(preds, target)
        ], dtype=torch.float, device=distances.device)

        ratio = torch.where(
            max_lengths == 0,
            torch.zeros_like(distances).float(),
            distances.float() / max_lengths
        )

        nls_values = torch.where(
        ratio <= 0.5,
        1 - ratio,
        torch.zeros_like(ratio),
    )

        if self.reduction == "none" or self.reduction is None:
            self.nls_values_list.append(nls_values)
        else:
            self.nls_score += nls_values.sum()
            self.num_elements += nls_values.shape[0]

    def _compute(
        self,
        nls_score: Tensor,
        num_elements: Union[Tensor, int],
    ) -> Tensor:
        """Compute the ANLS over state."""
        if nls_score.numel() == 0 or num_elements == 0:
            return torch.tensor(0, dtype=torch.int32)
        if self.reduction == "mean":
            return nls_score.sum() / num_elements
        if self.reduction == "sum":
            return nls_score.sum()
        if self.reduction is None or self.reduction == "none":
            return nls_score

    def compute(self) -> torch.Tensor:
        """Compute the NLS over state."""
        if self.reduction == "none" or self.reduction is None:
            return self._compute(dim_zero_cat(self.nls_values_list), 1)
        return self._compute(self.nls_score, self.num_elements)

if __name__ == "__main__":
    anls = NormalizedLevenshteinSimilarity(reduction='mean')
    preds = ["rain", "lnaguaeg"]
    target = ["shine", "language"]
    print(anls(preds, target))

最后

这份程式示范单一答案、逐对字串的 ANLS 计算。要与特定资料集的官方评分一致,还需核对大小写与空白正规化、多答案选取方式,以及空字串的处理规则。