windlass.providers.evaluation.external

external

RAGAS and DeepEval adapters.

Both are established evaluation frameworks with metric implementations that have been validated against human judgement. Windlass wraps them so you can run their metrics on Windlass samples without learning their data models.

Install with::

pip install "windlass[evaluation]"

Example

from windlass import Windlass # doctest: +SKIP ev = Windlass.evaluator("ragas", metrics=["faithfulness"], # doctest: +SKIP ... llm=Windlass.llm("openai")) # doctest: +SKIP ev.evaluate(answers) # doctest: +SKIP

RagasEvaluator

RagasEvaluator(
    *,
    metrics: Sequence[str] | None = None,
    threshold: float = 0.5,
    llm: Any = None,
    embeddings: Any = None,
    **config: Any
)

Bases: Evaluator

Evaluation via the RAGAS framework.

RAGAS specialises in reference-free RAG evaluation — it scores faithfulness and context quality without needing ground-truth answers, which is what makes it usable on production traffic rather than only on a curated test set.

Parameters:

Name	Type	Description	Default
metrics	Sequence[str] | None	RAGAS metric names — faithfulness, answer_relevancy, context_precision, context_recall, answer_correctness, answer_similarity.	None
threshold	float	Score at or above which a result passes.	0.5
llm	Any	Judge model. RAGAS needs a LangChain-compatible model; a Windlass LLM whose native() is LangChain-shaped is unwrapped automatically.	None
embeddings	Any	Embedding model for similarity-based metrics.	None
**config	Any	Forwarded to :class:~windlass.interfaces.evaluator.Evaluator.	{}

Raises:

Type	Description
MissingDependencyError	When ragas is not installed.
EvaluationError	When a metric name is unknown or the run fails.

Note

RAGAS evaluates a whole dataset at once, so this adapter overrides :meth:aevaluate rather than scoring sample by sample.

Source code in src\windlass\providers\evaluation\external.py

def __init__(
    self,
    *,
    metrics: Sequence[str] | None = None,
    threshold: float = 0.5,
    llm: Any = None,
    embeddings: Any = None,
    **config: Any,
) -> None:
    super().__init__(metrics=metrics, threshold=threshold, llm=llm, **config)
    self._ragas = require("ragas", extra="evaluation", feature="The RAGAS evaluator")
    self.embeddings = embeddings

default_metrics classmethod

default_metrics() -> tuple[str, ...]

Return RAGAS's core reference-free metrics.

Source code in src\windlass\providers\evaluation\external.py

@classmethod
def default_metrics(cls) -> tuple[str, ...]:
    """Return RAGAS's core reference-free metrics."""
    return ("faithfulness", "answer_relevancy", "context_precision")

available_metrics classmethod

available_metrics() -> tuple[str, ...]

Return every RAGAS metric this adapter can resolve.

Source code in src\windlass\providers\evaluation\external.py

@classmethod
def available_metrics(cls) -> tuple[str, ...]:
    """Return every RAGAS metric this adapter can resolve."""
    return (
        "faithfulness",
        "answer_relevancy",
        "context_precision",
        "context_recall",
        "answer_correctness",
        "answer_similarity",
    )

native

native() -> Any

Return the imported ragas module (Level 3 access).

Source code in src\windlass\providers\evaluation\external.py

def native(self) -> Any:
    """Return the imported ``ragas`` module (Level 3 access)."""
    return self._ragas

aevaluate_sample async

aevaluate_sample(sample: EvalSample) -> list[EvaluationResult]

Evaluate a single sample by running the dataset path with one row.

Source code in src\windlass\providers\evaluation\external.py

async def aevaluate_sample(self, sample: EvalSample) -> list[EvaluationResult]:
    """Evaluate a single sample by running the dataset path with one row."""
    report = await self.aevaluate([sample])
    return report.results

aevaluate async

aevaluate(samples: Sequence[EvalSample | Any]) -> EvaluationReport

Run RAGAS over the whole dataset.

Parameters:

Name	Type	Description	Default
samples	Sequence[EvalSample | Any]	Samples, RAG answers, or dicts.	required

Returns:

Type	Description
EvaluationReport	The aggregated report.

Raises:

Type	Description
EvaluationError	When RAGAS fails or produces no scores.

Source code in src\windlass\providers\evaluation\external.py

async def aevaluate(self, samples: Sequence[EvalSample | Any]) -> EvaluationReport:
    """Run RAGAS over the whole dataset.

    Args:
        samples: Samples, RAG answers, or dicts.

    Returns:
        The aggregated report.

    Raises:
        EvaluationError: When RAGAS fails or produces no scores.
    """
    prepared = [self._coerce(s) for s in samples]
    if not prepared:
        return EvaluationReport(samples=0)

    metrics = self._resolve_metrics()
    dataset = {
        "question": [s.question for s in prepared],
        "answer": [s.answer for s in prepared],
        "contexts": [s.contexts or [""] for s in prepared],
        "ground_truth": [s.reference for s in prepared],
    }

    def _run() -> Any:
        datasets = require("datasets", extra="evaluation", feature="The RAGAS evaluator")
        hf_dataset = datasets.Dataset.from_dict(dataset)
        kwargs: dict[str, Any] = {"dataset": hf_dataset, "metrics": metrics}
        judge = _langchain_model(self.llm)
        if judge is not None:
            kwargs["llm"] = judge
        if self.embeddings is not None:
            kwargs["embeddings"] = _langchain_model(self.embeddings) or self.embeddings
        return self._ragas.evaluate(**kwargs)

    try:
        outcome = await to_thread(_run)
    except Exception as exc:
        raise EvaluationError(
            f"RAGAS evaluation failed: {exc}",
            hint="RAGAS needs a LangChain-compatible judge model; pass "
            "llm=... explicitly if the default could not be unwrapped.",
        ) from exc

    return self._to_report(outcome, prepared)

DeepEvalEvaluator

DeepEvalEvaluator(
    *,
    metrics: Sequence[str] | None = None,
    threshold: float = 0.5,
    llm: Any = None,
    **config: Any
)

Bases: Evaluator

Evaluation via the DeepEval framework.

DeepEval's angle is unit-test-style evaluation: metrics carry thresholds and produce pass/fail verdicts with explanations, which fits naturally into CI.

Parameters:

Name	Type	Description	Default
metrics	Sequence[str] | None	DeepEval metric names — answer_relevancy, faithfulness, contextual_precision, contextual_recall, contextual_relevancy, hallucination, bias, toxicity.	None
threshold	float	Threshold passed to each metric.	0.5
llm	Any	Judge model. DeepEval accepts a model name string or its own DeepEvalBaseLLM; a Windlass LLM contributes its model name.	None
**config	Any	Forwarded to :class:~windlass.interfaces.evaluator.Evaluator.	{}

Raises:

Type	Description
MissingDependencyError	When deepeval is not installed.
EvaluationError	When a metric name is unknown.

Source code in src\windlass\providers\evaluation\external.py

def __init__(
    self,
    *,
    metrics: Sequence[str] | None = None,
    threshold: float = 0.5,
    llm: Any = None,
    **config: Any,
) -> None:
    super().__init__(metrics=metrics, threshold=threshold, llm=llm, **config)
    self._deepeval = require("deepeval", extra="evaluation", feature="The DeepEval evaluator")
    unknown = set(self.metrics) - set(self._METRIC_CLASSES)
    if unknown:
        raise EvaluationError(
            f"DeepEval has no metric named {', '.join(sorted(unknown))}.",
            hint=f"Available: {', '.join(self.available_metrics())}",
        )

default_metrics classmethod

default_metrics() -> tuple[str, ...]

Return DeepEval's core RAG metrics.

Source code in src\windlass\providers\evaluation\external.py

@classmethod
def default_metrics(cls) -> tuple[str, ...]:
    """Return DeepEval's core RAG metrics."""
    return ("answer_relevancy", "faithfulness")

available_metrics classmethod

available_metrics() -> tuple[str, ...]

Return every DeepEval metric this adapter can construct.

Source code in src\windlass\providers\evaluation\external.py

@classmethod
def available_metrics(cls) -> tuple[str, ...]:
    """Return every DeepEval metric this adapter can construct."""
    return tuple(cls._METRIC_CLASSES)

native

native() -> Any

Return the imported deepeval module (Level 3 access).

Source code in src\windlass\providers\evaluation\external.py

def native(self) -> Any:
    """Return the imported ``deepeval`` module (Level 3 access)."""
    return self._deepeval

aevaluate_sample async

aevaluate_sample(sample: EvalSample) -> list[EvaluationResult]

Score one sample with every configured DeepEval metric.

Parameters:

Name	Type	Description	Default
sample	EvalSample	The interaction to score.	required

Returns:

Type	Description
list[EvaluationResult]	One result per metric, carrying DeepEval's own explanation.

Raises:

Type	Description
EvaluationError	When DeepEval fails to construct or run a metric.

Source code in src\windlass\providers\evaluation\external.py

async def aevaluate_sample(self, sample: EvalSample) -> list[EvaluationResult]:
    """Score one sample with every configured DeepEval metric.

    Args:
        sample: The interaction to score.

    Returns:
        One result per metric, carrying DeepEval's own explanation.

    Raises:
        EvaluationError: When DeepEval fails to construct or run a metric.
    """
    from importlib import import_module

    def _run() -> list[EvaluationResult]:
        test_case_module = import_module("deepeval.test_case")
        metrics_module = import_module("deepeval.metrics")

        case = test_case_module.LLMTestCase(
            input=sample.question,
            actual_output=sample.answer,
            expected_output=sample.reference or None,
            retrieval_context=sample.contexts or None,
            context=sample.contexts or None,
        )

        produced: list[EvaluationResult] = []
        for name in self.metrics:
            metric_class = getattr(metrics_module, self._METRIC_CLASSES[name])
            kwargs: dict[str, Any] = {"threshold": self.threshold}
            model_name = _model_name(self.llm)
            if model_name:
                kwargs["model"] = model_name
            metric = metric_class(**kwargs)
            metric.measure(case)
            produced.append(
                EvaluationResult(
                    metric=name,
                    score=float(metric.score or 0.0),
                    passed=bool(metric.is_successful()),
                    threshold=self.threshold,
                    reason=str(getattr(metric, "reason", ""))[:500],
                    sample_id=sample.id,
                )
            )
        return produced

    try:
        return await to_thread(_run)
    except Exception as exc:
        raise EvaluationError(f"DeepEval evaluation failed: {exc}") from exc