Let’s look at the base class for all translators:
File: libs/langchain/langchain/chains/query_constructor/ir.py
class Visitor(ABC):
"""Defines interface for IR translation using visitor pattern."""
allowed_comparators: Optional[Sequence[Comparator]] = None
allowed_operators: Optional[Sequence[Operator]] = None
def _validate_func(self, func: Union[Operator, Comparator]) -> None:
if isinstance(func, Operator) and self.allowed_operators is not None:
if func not in self.allowed_operators:
raise ValueError(
f"Received disallowed operator {func}. Allowed "
f"comparators are {self.allowed_operators}"
)
if isinstance(func, Comparator) and self.allowed_comparators is not None:
if func not in self.allowed_comparators:
raise ValueError(
f"Received disallowed comparator {func}. Allowed "
f"comparators are {self.allowed_comparators}"
)
@abstractmethod
def visit_operation(self, operation: Operation) -> Any:
"""Translate an Operation."""
@abstractmethod
def visit_comparison(self, comparison: Comparison) -> Any:
"""Translate a Comparison."""
@abstractmethod
def visit_structured_query(self, structured_query: StructuredQuery) -> Any:
"""Translate a StructuredQuery."""
allowed_comparators and allowed_operators represent operators and comparators that our final query accepts.
_validate_func checks whether operator/comparator is allowed, ie., whether it is of the correct type and/or allowed. Otherwise, an error is thrown.
I have done my implementation of the MilvusTranslator in accordance to the query documentation. In order to perform a Similarity Search we need a vector and boolean expression.
We get the vector after embedding our query text in order to find most similar database objects, and this is why we need to feed embeddings into the Milvus (please see previous tutorial).
Right now we are focusing on translating the boolean expression. We need to translate the data that gets passed to the similarity_search method inside the Milvus class:
File: libs/langchain/langchain/vectorstores/milvus.py
def similarity_search(
self,
query: str,
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
"""Perform a similarity search against the query string.
Args:
query (str): The text to search.
k (int, optional): How many results to return. Defaults to 4.
param (dict, optional): The search params for the index type.
Defaults to None.
expr (str, optional): Filtering expression. Defaults to None.
timeout (int, optional): How long to wait before timeout error.
Defaults to None.
kwargs: Collection.search() keyword arguments.
Returns:
List[Document]: Document results for search.
"""
if self.col is None:
logger.debug("No existing collection to search.")
return []
res = self.similarity_search_with_score(
query=query, k=k, param=param, expr=expr, timeout=timeout, **kwargs
)
return [doc for doc, _ in res]
The grammar needs to follow the Boolean Expression rules.
This is my implementation of the whole class:
"""Logic for converting internal query language to a valid Milvus query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
Visitor,
)
COMPARATOR_TO_BER = {
Comparator.EQ: "==",
Comparator.GT: ">",
Comparator.GTE: ">=",
Comparator.LT: "<",
Comparator.LTE: "<=",
}
UNARY_OPERATORS = [Operator.NOT]
def process_value(value: Union[int, float, str]) -> str:
# required for comparators involving strings
if isinstance(value, str):
# If the value is already a string, add double quotes
return f'"{value}"'
else:
# If the valueis not a string, convert it to a string without double quotes
return str(value)
class MilvusTranslator(Visitor):
"""Translate Milvus internal query language elements to valid filters."""
"""Subset of allowed logical operators."""
allowed_operators = [Operator.AND, Operator.NOT, Operator.OR]
"""Subset of allowed logical comparators."""
allowed_comparators = [
Comparator.EQ,
Comparator.GT,
Comparator.GTE,
Comparator.LT,
Comparator.LTE,
]
def _format_func(self, func: Union[Operator, Comparator]) -> str:
self._validate_func(func)
value = func.value
if isinstance(func, Comparator):
value = COMPARATOR_TO_BER[func]
return f"{value}"
def visit_operation(self, operation: Operation) -> str:
if operation.operator in UNARY_OPERATORS and len(operation.arguments) == 1:
operator = self._format_func(operation.operator)
return operator + "(" + operation.arguments[0].accept(self) + ")"
elif operation.operator in UNARY_OPERATORS:
raise ValueError(
f'"{operation.operator.value}" can have only one argument in Milvus'
)
else:
args = [arg.accept(self) for arg in operation.arguments]
operator = self._format_func(operation.operator)
return "(" + (" " + operator + " ").join(args) + ")"
def visit_comparison(self, comparison: Comparison) -> str:
comparator = self._format_func(comparison.comparator)
processed_value = process_value(comparison.value)
attribute = comparison.attribute
return "( " + attribute + " " + comparator + " " + processed_value + " )"
def visit_structured_query(
self, structured_query: StructuredQuery
) -> Tuple[str, dict]:
if structured_query.filter is None:
kwargs = {}
else:
kwargs = {"expr": structured_query.filter.accept(self)}
return structured_query.query, kwargs