Source code for neer_match_utilities.embeddings

"""
Similarity of field values from a pretrained text encoder (sentence embeddings).

Each distinct value of the chosen columns is encoded once by a pretrained model
(by default ``BAAI/bge-m3``, multilingual); the similarity of two values is the
cosine of their vectors. It is registered like any other similarity, so it can
be named in the similarity map, and then the logit, probit and gradient-boosting
models (through :class:`~neer_match_utilities.similarity_features.SimilarityFeatures`),
the neural network of neer_match (through its ``SimilarityEncoder``) and the
feature selection all use it.

The vectors are kept in a cache folder. Encoding needs the optional packages
``sentence-transformers`` and ``torch`` (``pip install
"neer-match-utilities[embeddings]"``); reading a cache needs only numpy, so the
models can be trained in an environment without torch, and the cache is what a
replication package stores.

Encoding runs in a Python process of its own (``_encode_worker.py``), which
loads neither this package nor TensorFlow: on Linux with a GPU, TensorFlow and
triton (loaded by sentence-transformers) cannot be loaded into one process in
this order.
"""
from __future__ import annotations

import importlib.util
import json
import os
import subprocess
import sys
import tempfile
import warnings
from datetime import datetime, timezone
from pathlib import Path
from typing import Callable, Sequence

import numpy as np
import pandas as pd

from .custom_similarities import register_similarities

DEFAULT_MODEL = "BAAI/bge-m3"
_VALUES, _VECTORS, _META = "values.json", "vectors.npy", "meta.json"
_WORKER = Path(__file__).with_name("_encode_worker.py")


def _sentence_transformers_available() -> bool:
    return importlib.util.find_spec("sentence_transformers") is not None


def _is_missing(value) -> bool:
    """None, NaN, pandas.NA, or a string that is empty after stripping."""
    if value is None:
        return True
    if isinstance(value, str):
        return not value.strip()
    try:
        return bool(pd.isna(value))
    except (TypeError, ValueError):
        return False


def _atomic_write(path: Path, write: Callable) -> None:
    """Write to a temporary file in the same folder, then replace `path`."""
    fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.")
    try:
        with os.fdopen(fd, "wb") as f:
            write(f)
        os.replace(tmp, path)
    except BaseException:
        if os.path.exists(tmp):
            os.remove(tmp)
        raise


def _resolve_revision(model_name: str) -> str | None:
    """Commit of the model on the Hugging Face Hub, or None if it cannot be found."""
    if Path(model_name).exists():
        return None
    try:
        from huggingface_hub import HfApi
        return HfApi().model_info(model_name).sha
    except Exception:
        warnings.warn(
            f"Could not look up the current commit of {model_name!r}; the cache records "
            "no revision. Pass revision= to pin the model.",
            stacklevel=3,
        )
        return None


[docs] class EmbeddingSimilarity: """ Cosine similarity of sentence embeddings, available to neer_match by name. Parameters ---------- cache_dir : str or Path Folder for the vectors (``vectors.npy``), the encoded values (``values.json``) and the model they come from (``meta.json``). If it holds a cache already, that cache is read, and only values not in it are encoded later. The values are the data themselves: store the folder under the same terms as the data. model_name : str, optional Model on the Hugging Face Hub, or a local path, loaded with ``sentence_transformers.SentenceTransformer``. Default ``"BAAI/bge-m3"``. A cache made with another model raises ValueError. revision : str or None, optional Commit of the model to use. Default None: the commit recorded in the cache, or else the current one on the Hub, which is then recorded. similarity_name : str, optional Name under which the similarity is registered and used in the similarity map. Default ``"embedding_cosine"``. Use one name per model if several are compared. batch_size : int, optional Values encoded at once. Default 64. max_seq_length : int, optional Tokens per value; longer values are cut. Default 512 (bge-m3 accepts up to 8192, which field values rarely need). device : str or None, optional Device for encoding, e.g. ``"cuda"`` or ``"cpu"``. Default None: the GPU if one is available. fp16 : bool, optional Encode in half precision (GPU only). Default False. cuda_visible_devices : str or None, optional GPUs the encoding process may use, as in the environment variable ``CUDA_VISIBLE_DEVICES`` (e.g. ``"1"``). Default None: the same GPUs as this process. Lets this process hide the GPUs from TensorFlow (``CUDA_VISIBLE_DEVICES=``), which otherwise may take a GPU's memory when the package is imported, while the model encodes on one. encoder : callable or None, optional Instead of the model: a function that takes a list of strings and returns one vector per string (e.g. for tests or another library). Default None. Notes ----- - A value is looked up exactly as it appears in the data (as ``str(value)``), so encode the columns in the form they have when the similarities are computed, i.e. after ``Prepare.format``, and encode again after adding rows, e.g. with ``synth_matches`` or ``synth_mismatches``: only the new values are encoded. A value that was not encoded raises KeyError. - Missing values, and strings that are empty, have similarity 0, as in :class:`~neer_match_utilities.similarity_features.SimilarityFeatures`. - The cosine is cut to [0, 1], the range of the other similarities. - The vectors are normalized and kept as float32: 4 bytes times the dimension (1024 for bge-m3) per distinct value. - To use a saved model later, create the object again from the cache before predicting, so that the similarity is registered. - Each call of :meth:`encode` that finds new values starts a Python process that loads the model, encodes them and ends (see the module notes); encode all frames in one call where possible. """
[docs] def __init__( self, cache_dir: str | Path, model_name: str = DEFAULT_MODEL, *, revision: str | None = None, similarity_name: str = "embedding_cosine", batch_size: int = 64, max_seq_length: int = 512, device: str | None = None, fp16: bool = False, cuda_visible_devices: str | None = None, encoder: Callable[[list], np.ndarray] | None = None, ): self.cache_dir = Path(cache_dir) self.model_name = model_name self.revision = revision self.similarity_name = similarity_name self.batch_size = batch_size self.max_seq_length = max_seq_length self.device = device self.fp16 = fp16 self.cuda_visible_devices = cuda_visible_devices self._encoder = encoder self._values: list[str] = [] self._index: dict[str, int] = {} self._vectors = np.zeros((0, 0), dtype=np.float32) self._rows: list[np.ndarray] = [] if (self.cache_dir / _META).exists(): self._read_cache() register_similarities({similarity_name: self.similarity})
# ------------------------------------------------------------------ data
[docs] def __len__(self) -> int: return len(self._values)
@property def dim(self) -> int: """Length of the vectors (0 before anything is encoded).""" return int(self._vectors.shape[1]) if len(self) else 0 @property def values(self) -> list: """The encoded values, in the order of the rows of :attr:`vectors`.""" return list(self._values) @property def vectors(self) -> np.ndarray: """The normalized vectors, one row per encoded value.""" return self._vectors # ------------------------------------------------------------ similarity
[docs] def similarity(self, x, y) -> float: """Cosine of the vectors of `x` and `y`, cut to [0, 1]; 0 if either is missing.""" if _is_missing(x) or _is_missing(y): return 0.0 try: i, j = self._index[str(x)], self._index[str(y)] except KeyError as e: raise KeyError( f"{e.args[0]!r} has no embedding in {self.cache_dir}: encode the data " "that contain it first (also after rows were added, e.g. by " "synth_matches or synth_mismatches)." ) from None return min(max(float(np.dot(self._rows[i], self._rows[j])), 0.0), 1.0)
# -------------------------------------------------------------- encoding
[docs] def encode(self, *frames: pd.DataFrame, columns: Sequence[str]) -> "EmbeddingSimilarity": """ Encode the values of `columns` in `frames` that are not in the cache yet. Parameters ---------- *frames : pandas.DataFrame Data whose values are encoded, e.g. ``left, right``. Each must have all `columns`; if the left and right names differ, call once per frame. columns : sequence of str Columns to encode. Returns ------- EmbeddingSimilarity The object itself; the cache folder is updated if anything was new. """ new: list[str] = [] seen = set(self._index) for frame in frames: missing = [c for c in columns if c not in frame.columns] if missing: raise KeyError(f"Columns not in the data: {missing}") for col in columns: for value in pd.unique(frame[col].to_numpy(dtype=object)): if _is_missing(value): continue key = str(value) if key not in seen: seen.add(key) new.append(key) if new: self._append(new, self._encode_texts(new)) self._write_cache() return self
def _encode_texts(self, texts: list[str]) -> np.ndarray: if self._encoder is not None: vectors = self._encoder(texts) else: vectors = self._encode_in_subprocess(texts) vectors = np.asarray(vectors, dtype=np.float32) if vectors.ndim != 2 or vectors.shape[0] != len(texts): raise ValueError( f"The encoder returned shape {vectors.shape} for {len(texts)} values; " "expected one vector per value." ) if len(self) and vectors.shape[1] != self.dim: raise ValueError(f"The encoder returned vectors of length {vectors.shape[1]}, the cache has {self.dim}.") norms = np.linalg.norm(vectors, axis=1, keepdims=True) return vectors / np.where(norms > 0, norms, 1.0) def _encode_in_subprocess(self, texts: list[str]) -> np.ndarray: """Encode `texts` with the model in a Python process of its own (see the module notes).""" if not _sentence_transformers_available(): raise ImportError( "Encoding needs sentence-transformers and torch: " 'pip install "neer-match-utilities[embeddings]". ' "Reading an existing cache does not." ) if self.revision is None: self.revision = _resolve_revision(self.model_name) self.cache_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=self.cache_dir, prefix=".encode-") as tmp: job, out = Path(tmp) / "job.json", Path(tmp) / "vectors.npy" job.write_text(json.dumps({ "model_name": self.model_name, "revision": self.revision, "device": self.device, "fp16": self.fp16, "max_seq_length": self.max_seq_length, "batch_size": self.batch_size, "texts": texts, "out": str(out), }, ensure_ascii=False), encoding="utf-8") # -I: the worker's folder (this package) is not put on sys.path env = os.environ.copy() if self.cuda_visible_devices is not None: env["CUDA_VISIBLE_DEVICES"] = str(self.cuda_visible_devices) done = subprocess.run([sys.executable, "-I", str(_WORKER), str(job)], env=env) if done.returncode != 0 or not out.exists(): raise RuntimeError( f"Encoding {len(texts)} values with {self.model_name!r} failed " f"(exit code {done.returncode}); see the output above." ) return np.load(out) def _append(self, texts: list[str], vectors: np.ndarray) -> None: start = len(self._values) self._values.extend(texts) self._index.update({t: start + k for k, t in enumerate(texts)}) self._vectors = vectors if start == 0 else np.vstack([self._vectors, vectors]) self._rows = list(self._vectors) # ----------------------------------------------------------------- cache def _meta(self) -> dict: try: from importlib.metadata import version st_version = version("sentence-transformers") except Exception: st_version = None return { "model_name": self.model_name, "revision": self.revision, "encoder": "sentence-transformers" if self._encoder is None else "custom", "sentence_transformers": st_version if self._encoder is None else None, "dim": self.dim, "max_seq_length": self.max_seq_length, "normalized": True, "n_values": len(self), "updated": datetime.now(timezone.utc).isoformat(timespec="seconds"), } def _write_cache(self) -> None: self.cache_dir.mkdir(parents=True, exist_ok=True) _atomic_write(self.cache_dir / _VECTORS, lambda f: np.save(f, self._vectors)) _atomic_write( self.cache_dir / _VALUES, lambda f: f.write(json.dumps(self._values, ensure_ascii=False).encode("utf-8")), ) _atomic_write( self.cache_dir / _META, lambda f: f.write(json.dumps(self._meta(), indent=2).encode("utf-8")), ) def _read_cache(self) -> None: meta = json.loads((self.cache_dir / _META).read_text(encoding="utf-8")) if meta.get("model_name") != self.model_name: raise ValueError( f"The cache in {self.cache_dir} was made with {meta.get('model_name')!r}, " f"not {self.model_name!r}. Use another cache_dir per model." ) cached_revision = meta.get("revision") if self.revision is not None and cached_revision is not None and self.revision != cached_revision: raise ValueError( f"The cache in {self.cache_dir} was made with revision {cached_revision!r} " f"of {self.model_name!r}, not {self.revision!r}." ) self.revision = self.revision or cached_revision values = json.loads((self.cache_dir / _VALUES).read_text(encoding="utf-8")) vectors = np.load(self.cache_dir / _VECTORS) if vectors.ndim != 2 or len(values) != vectors.shape[0]: raise ValueError( f"The cache in {self.cache_dir} is inconsistent: {len(values)} values, " f"vectors of shape {vectors.shape}." ) self._values, self._index = [], {} self._append(values, vectors.astype(np.float32, copy=False)) # --------------------------------------------------------------- pickling def __getstate__(self): state = self.__dict__.copy() state["_rows"] = None return state def __setstate__(self, state): self.__dict__.update(state) self._rows = list(self._vectors)