实体关系抽取

12@dataclass13class Entity:14    """Entity representation."""1516    text: str17    label: str18    start_char: int19    end_char: int20    confidence: float = 1.021    metadata: Dict[str, Any] = field(default_factory=dict)222324@dataclass25class Relation:26    """Relation representation."""2728    subject: Entity29    predicate: str30    object: Entity31    confidence: float = 1.032    context: str = ""33    metadata: Dict[str, Any] = field(default_factory=dict)343536@dataclass37class Triplet:38    """RDF triplet representation."""3940    subject: str41    predicate: str42    object: str43    confidence: float = 1.044    metadata: Dict[str, Any] = field(default_factory=dict)4546    def get(self, key: str, default: Any = None) -> Any:47        """Get attribute value like a dictionary."""48        return getattr(self, key, default)4950    def __getitem__(self, key: str) -> Any:51        """Get item like a dictionary."""52        return getattr(self, key)

57class NamedEntityRecognizer:58    """59    Named entity recognition handler.6061    • Extracts named entities from text62    • Classifies entities by type and category63    • Provides confidence scores for entities64    • Supports custom entity types65    • Handles multiple languages and domains66    • Processes batch text collections67    """68    def __init__(69        self,70        methods: Optional[List[str]] = None,71        confidence_threshold: float = 0.5,72        merge_overlapping: bool = True,73        include_standard_types: bool = True,74        method=None,75        config=None,76        **kwargs77    ):78        """79        Initialize named entity recognizer.8081        Args:82            methods: Extraction methods to use (e.g., ["spacy", "rule-based"])83            confidence_threshold: Minimum confidence score (0.0-1.0)84            merge_overlapping: Whether to merge overlapping entities85            include_standard_types: Include standard types (Person, Org, Location)86            method: Extraction method(s) - passed to NERExtractor (legacy)87            config: Legacy config dict (deprecated, use kwargs)88            **kwargs: Additional configuration options passed to NERExtractor89        """90        self.logger = get_logger("named_entity_recognizer")91        self.config = config or {}92        self.config.update(kwargs)93        self.progress_tracker = get_progress_tracker()94        # Ensure progress tracker is enabled95        if not self.progress_tracker.enabled:96            self.progress_tracker.enabled = True9798        # Store parameters99        self.methods = methods or ["spacy"]100        self.confidence_threshold = confidence_threshold101        self.merge_overlapping = merge_overlapping102        self.include_standard_types = include_standard_types103104        # Use NERExtractor for actual extraction105        ner_config = self.config.get("ner", {})106        ner_config["confidence_threshold"] = confidence_threshold107        ner_config["min_confidence"] = confidence_threshold108        ner_config["merge_overlapping"] = merge_overlapping109        if method is not None:110            ner_config["method"] = method111        elif methods:112            ner_config["method"] = methods[0] if len(methods) == 1 else methods113        self.ner_extractor = NERExtractor(**ner_config, **self.config)114        self.entity_classifier = EntityClassifier(**self.config.get("classifier", {}))115        self.confidence_scorer = EntityConfidenceScorer(**self.config.get("scorer", {}))

191class EntityClassifier:192    """Entity classification engine."""193194    def __init__(self, **config):195        """Initialize entity classifier."""196        self.logger = get_logger("entity_classifier")197        self.config = config198199        # Entity type mappings200        self.type_hierarchy = {201            "PERSON": ["PERSON", "PER"],202            "ORG": ["ORG", "ORGANIZATION"],203            "GPE": ["GPE", "LOCATION", "LOC"],204            "DATE": ["DATE", "TIME"],205            "MONEY": ["MONEY", "CURRENCY"],206            "PERCENT": ["PERCENT", "PERCENTAGE"],207        }208209    def classify_entity_type(self, entity: Entity, **context) -> str:210        """211        Classify entity by type.212213        Args:214            entity: Entity to classify215            **context: Context information216217        Returns:218            str: Entity type219        """220        # Normalize entity type221        label = entity.label.upper()222223        # Check type hierarchy224        for canonical_type, variants in self.type_hierarchy.items():225            if label in variants:226                return canonical_type227228        return label229230    def disambiguate_entity(231        self, entity: Entity, candidates: List[Entity], **context232    ) -> Optional[Entity]:233        """234        Disambiguate entity among candidates.235236        Args:237            entity: Entity to disambiguate238            candidates: Candidate entities239            **context: Context information240241        Returns:242            Entity: Best matching entity or None243        """244        if not candidates:245            return None246247        # Simple disambiguation by type match248        entity_type = entity.label249        matching = [c for c in candidates if c.label == entity_type]250251        if matching:252            # Return first match with highest confidence253            return max(matching, key=lambda e: e.confidence)254255        return candidates[0] if candidates else None256257    def classify_entities(258        self, entities: List[Entity], **context259    ) -> Dict[str, List[Entity]]:260        """261        Classify entities by type.262263        Args:264            entities: List of entities265            **context: Context information266267        Returns:268            dict: Entities grouped by type269        """270        classified = {}271        for entity in entities:272            entity_type = self.classify_entity_type(entity, **context)273            if entity_type not in classified:274                classified[entity_type] = []275            classified[entity_type].append(entity)276277        return classified

362            # Try each method in order (fallback chain)363            all_entities = []364            for method_name in methods:365                try:366                    self.progress_tracker.update_tracking(367                        tracking_id,368                        message=f"Extracting entities using {method_name}...",369                    )370                    method_func = get_entity_method(method_name)371372                    # Prepare method-specific options373                    method_options = all_options.copy()374                    if method_name == "huggingface":375                        # Prioritize runtime options over config/defaults376                        method_options["model"] = (377                            options.get("huggingface_model") 378                            or options.get("model") 379                            or self.huggingface_model380                        )381                        method_options["device"] = all_options.get("device")382                    elif method_name == "llm":383                        method_options["provider"] = all_options.get(384                            "provider", "openai"385                        )386                        method_options["model"] = all_options.get(387                            "llm_model", all_options.get("model")388                        )389                        # Ensure api_key is populated: check explicitly provided or fallback to env390                        current_key = method_options.get("api_key")391                        if not current_key:392                            # Not found or empty/None, try environment393                            import os394                            provider_name = method_options.get("provider", "openai")395                            env_key = f"{provider_name.upper()}_API_KEY"396                            api_key = os.getenv(env_key)397                            if api_key:398                                method_options["api_key"] = api_key399400                    entities = method_func(text, **method_options)401402                    # Apply weighted scoring if entity_types are provided403                    if entity_types:404                        try:405                            from .methods import calculate_weighted_confidence406                            for e in entities:407                                e.confidence = calculate_weighted_confidence(408                                    item_type=e.label,409                                    original_confidence=e.confidence,410                                    valid_types=entity_types,411                                    item_text=e.text412                                )413                        except ImportError:414                            pass415416                    # Filter by confidence417                    filtered = [e for e in entities if e.confidence >= min_confidence]418                    419                    if filtered:420                        all_entities.append((method_name, filtered))421422                        # If not using ensemble, return first successful result423                        if not self.ensemble_voting:424                            # Ensure default metadata425                            for e in filtered:426                                if e.metadata is None: e.metadata = {}427                                if "batch_index" not in e.metadata: e.metadata["batch_index"] = 0428429                            self.progress_tracker.stop_tracking(430                                tracking_id,431                                status="completed",432                                message=f"Extracted {len(filtered)} entities using {method_name}",433                            )434                            return filtered435436                except Exception as e:437                    self.logger.warning(438                        "Method %s failed: %s", method_name, e, exc_info=True439                    )440                    continue441442            # Ensemble voting if enabled443            if self.ensemble_voting and len(all_entities) > 1:444                entities = self._vote_entities(445                    [entities for _, entities in all_entities]446                )447            elif all_entities:448                entities = all_entities[0][1]  # Use first successful method449            else:450                # Fallback to pattern-based extraction if all models fail451                entities = self._extract_fallback(text)

81class RelationExtractor:82    """Relation extractor for entity relationships."""83    def __init__(84        self,85        method: Union[str, List[str]] = "pattern",86        relation_types: Optional[List[str]] = None,87        bidirectional: bool = False,88        confidence_threshold: float = 0.6,89        max_distance: int = 50,90        **config91    ):92        """93        Initialize relation extractor.9495        Args:96            method: Extraction method(s). Can be:97                - "pattern": Pattern-based extraction (default)98                - "regex": Regex-based extraction99                - "cooccurrence": Co-occurrence based100                - "dependency": Dependency parsing based101                - "huggingface": HuggingFace model102                - "llm": LLM-based extraction103                - List of methods for fallback chain104            relation_types: Specific relation types to extract (e.g., ["founded", "works_at"])105            bidirectional: Whether to extract bidirectional relations106            confidence_threshold: Minimum confidence score (0.0-1.0)107            max_distance: Maximum token distance between entities108            **config: Additional configuration options:109                - model: Model name (for dependency/HuggingFace methods)110                - huggingface_model: HuggingFace model name111                - provider: LLM provider (for LLM method)112                - llm_model: LLM model name113                - device: Device for HuggingFace models114                - validate: Enable validation (default: False)115        """116        self.logger = get_logger("relation_extractor")117        self.config = config118        self.progress_tracker = get_progress_tracker()119        # Ensure progress tracker is enabled120        if not self.progress_tracker.enabled:121            self.progress_tracker.enabled = True122123        # Store parameters124        self.relation_types = relation_types125        self.bidirectional = bidirectional126        self.confidence_threshold = confidence_threshold127        self.max_distance = max_distance128        self.verbose = config.get("verbose", False)129130        # Method configuration131        self.method = method if isinstance(method, list) else [method]132        self.min_confidence = config.get("min_confidence", confidence_threshold)133        self.validate = config.get("validate", False)134135        # Common relation patterns136        # Entity pattern allowing for dots and spaces (e.g., "Apple Inc.", "New York")137        ent_pat = r"[\w\.]+(?:\s+[\w\.]+)*"⋯        # ... 空行略 ...139        self.relation_patterns = {140            "founded_by": [141                # Subject founded by Object142                fr"(?P<subject>[\w\.\s]+?)\s+(?:was\s+)?founded\s+by\s+(?P<object>{ent_pat})",143                # Object founded Subject144                fr"(?P<object>{ent_pat})\s+founded\s+(?P<subject>{ent_pat})",145            ],146            "located_in": [147                fr"(?P<subject>[\w\.\s]+?)\s+is\s+located\s+in\s+(?P<object>{ent_pat})",148                fr"(?P<subject>[\w\.\s]+?)\s+in\s+(?P<object>{ent_pat})",149            ],150            "works_for": [151                fr"(?P<subject>[\w\.\s]+?)\s+works?\s+for\s+(?P<object>{ent_pat})",152                fr"(?P<subject>[\w\.\s]+?)\s+is\s+an?\s+employee\s+of\s+(?P<object>{ent_pat})",153            ],154            "born_in": [155                fr"(?P<subject>[\w\.\s]+?)\s+was\s+born\s+in\s+(?P<object>{ent_pat})",156                fr"(?P<subject>[\w\.\s]+?)\s+born\s+in\s+(?P<object>{ent_pat})",157            ],158        }

477            # Use first successful method or combine478            if all_relations:479                relations = all_relations[0][1]  # Use first successful method480            else:481                # Fallback to pattern-based extraction if all models fail482                relations = self._extract_with_patterns(text, entities)⋯                # ... 空行略 ...484                # Last resort: if patterns also fail but we have entities, force some relations485                if not relations and entities and len(entities) >= 2:486                     relations = self._extract_last_resort_relations(text, entities)487488            # Validate if enabled489            if validate:490                relations = self.validate_relations(relations)491492            self.progress_tracker.stop_tracking(493                tracking_id,494                status="completed",495                message=f"Extracted {len(relations)} relations",496            )497            return relations

527            # Use first successful method or fallback to relation conversion528            if all_triplets:529                triplets = all_triplets[0][1]530            else:531                # Fallback: Convert relations to triplets532                if relations:533                    self.progress_tracker.update_tracking(534                        tracking_id,535                        message=f"Converting {len(relations)} relations to triplets...",536                    )537                    triplets = []538                    for relation in relations:539                        triplet = Triplet(540                            subject=self._format_uri(relation.subject.text),541                            predicate=self._format_uri(relation.predicate),542                            object=self._format_uri(relation.object.text),543                            confidence=relation.confidence,544                            metadata={"context": relation.context, **relation.metadata},545                        )546                        triplets.append(triplet)547                else:548                    # Last resort: Try rule-based extraction if no relations exist549                    self.progress_tracker.update_tracking(550                        tracking_id,551                        message="No relations found. Trying rule-based triplet extraction...",552                    )553                    method_func = get_triplet_method("rules")554                    triplets = method_func(text, entities=entities, relations=[], **all_options)555556            # Validate triplets557            if options.get("validate", self._should_validate):558                self.progress_tracker.update_tracking(559                    tracking_id, message="Validating triplets..."560                )561                triplets = self.triplet_validator.validate_triplets(triplets)562563            self.progress_tracker.stop_tracking(564                tracking_id,565                status="completed",566                message=f"Extracted {len(triplets)} triplets",567            )568            return triplets

691class TripletValidator:692    """Triplet validation engine."""693694    def __init__(self, **config):695        """Initialize triplet validator."""696        self.logger = get_logger("triplet_validator")697        self.config = config698699    def validate_triplet(self, triplet: Triplet, **criteria) -> bool:700        """701        Validate individual triplet.702703        Args:704            triplet: Triplet to validate705            **criteria: Validation criteria706707        Returns:708            bool: True if valid709        """710        # Check structure711        if not triplet.subject or not triplet.predicate or not triplet.object:712            return False713714        # Check confidence715        min_confidence = criteria.get("min_confidence", 0.5)716        if triplet.confidence < min_confidence:717            return False718719        return True720721    def validate_triplets(self, triplets: List[Triplet], **criteria) -> List[Triplet]:722        """723        Validate list of triplets.724725        Args:726            triplets: List of triplets727            **criteria: Validation criteria728729        Returns:730            list: Valid triplets731        """732        return [t for t in triplets if self.validate_triplet(t, **criteria)]

4class EntityOut(BaseModel):5    """Canonical schema for entity extraction output."""6    model_config = ConfigDict(populate_by_name=True, extra="ignore")⋯    # ... 空行略 ...·    text: str = Field(..., description="The text content of the entity")·    label: str = Field(..., description="The type or label of the entity (e.g., PERSON, ORG)")·    start: int = Field(0, description="Start character index", alias="start_char")·    end: int = Field(0, description="End character index", alias="end_char")·    confidence: float = Field(0.9, description="Confidence score between 0 and 1")·    metadata: dict = Field(default_factory=dict, description="Additional metadata including provenance")··    @field_validator("text", mode="before")·    @classmethod·    def clean_text(cls, v):·        if isinstance(v, str):·            return v.strip()·        return str(v)··    @field_validator("confidence", mode="before")·    @classmethod·    def normalize_confidence(cls, v):·        if isinstance(v, str):·            try:·                v = float(v)·            except ValueError:·                return 0.9·        if isinstance(v, (int, float)):·            return max(0.0, min(1.0, float(v)))·        return 0.9⋯    # ... 空行略 ...34    @model_validator(mode="before")35    @classmethod36    def handle_aliases(cls, data):37        if isinstance(data, dict):38            # Handle 'type' as alias for 'label'39            if "label" not in data and "type" in data:40                data["label"] = data["type"]41            # Handle 'value' or 'span' as alias for 'text'42            if "text" not in data:43                if "value" in data:44                    data["text"] = data["value"]45                elif "span" in data:46                    data["text"] = data["span"]47        return data

72    def _generate_key(self, text: str, **params) -> str:73        """74        Generate a stable cache key based on text and parameters.75        76        Note: Sensitive parameters like 'api_key' are excluded from the cache key77        to prevent security risks and ensure cache sharing where appropriate.78        """79        # Filter out sensitive keys80        sensitive_keys = {'api_key', 'token', 'password', 'secret', 'auth', 'authorization'}81        filtered_params = {k: v for k, v in params.items() if k.lower() not in sensitive_keys}8283        # Create a stable string representation of params84        # Sort keys to ensure consistent ordering85        param_str = json.dumps(filtered_params, sort_keys=True, default=str)⋯        # ... 空行略 ...·        # Combine text and params·        content = f"{text}|{param_str}"⋯        # ... 空行略 ...90        # Return hash (SHA-256 for better security than MD5)91        return hashlib.sha256(content.encode('utf-8')).hexdigest()

44class ProvenanceMixin:45    """46    Mixin to add provenance tracking to any extractor class.47    48    This mixin provides the common provenance infrastructure that can be49    added to any extraction class without modifying its core functionality.50    """⋯    # ... 空行略 ...52    def __init__(53        self,54        provenance: bool = False,55        agent_id: Optional[str] = None,56        is_automated: bool = True,57        **kwargs,58    ):59        """60        Initialize provenance tracking.6162        Args:63            provenance: Enable provenance tracking (default: False)64            agent_id: Agent identifier for accountability (issue #825); defaults65                to the wrapping class name66            is_automated: Whether this agent acted without direct human review67            **kwargs: Additional arguments passed to parent class68        """69        self.provenance = provenance70        self._prov_manager = None71        self._agent_id = agent_id or self.__class__.__name__72        self._is_automated = is_automated7374        if provenance:75            try:76                from semantica.provenance import ProvenanceManager77                self._prov_manager = ProvenanceManager()78            except ImportError:79                # Graceful degradation if provenance module not available80                self.provenance = False8182    def _track_extraction(83        self,84        entity_id: str,85        source: str,86        entity_type: str,87        **metadata88    ) -> None:89        """90        Track extraction with provenance.9192        Args:93            entity_id: Unique identifier for extracted entity94            source: Source document or text95            entity_type: Type of entity (e.g., 'named_entity', 'relation')96            **metadata: Additional metadata to track97        """98        if self.provenance and self._prov_manager:99            activity_started_at_time = metadata.pop("activity_started_at_time", None)100            activity_ended_at_time = metadata.pop("activity_ended_at_time", None)101            self._prov_manager.track_entity(102                entity_id=entity_id,103                source=source,104                entity_type=entity_type,105                agent_id=self._agent_id,106                agent_type="software_agent",107                is_automated=self._is_automated,108                activity_started_at_time=activity_started_at_time,109                activity_ended_at_time=activity_ended_at_time,110                metadata=metadata111            )

550def calculate_weighted_confidence(551    item_type: str, 552    original_confidence: float, 553    valid_types: Optional[List[str]] = None,554    item_text: Optional[str] = None,555    weight_method: float = 0.5,556    weight_similarity: float = 0.5557) -> float:558    """559    Calculate weighted confidence score using both Label and Content similarity.560    Final Score = (weight_method * original_confidence) + (weight_similarity * max(label_sim, content_sim))561    562    Args:563        item_type: The extracted type/label/predicate (e.g., "PERSON", "founded_by")564        original_confidence: The confidence score from the extraction method (0.0-1.0)565        valid_types: List of valid/preferred types provided by user566        item_text: The actual text content extracted (e.g., "Steve Jobs", "acquired")567        weight_method: Weight for the original method confidence (default 0.5)568        weight_similarity: Weight for the similarity score (default 0.5)569        570    Returns:571        float: Weighted confidence score (0.0-1.0)572    """573    if not valid_types:574        return original_confidence575        576    # Similarity 1: Label vs Valid Types (e.g., "PERSON" vs "Artist")577    label_similarity = calculate_similarity(item_type, valid_types)578    579    # Similarity 2: Content vs Valid Types (e.g., "Picasso" vs "Artist")580    content_similarity = 0.0581    if item_text:582        content_similarity = calculate_similarity(item_text, valid_types)583        584    # Take the best similarity match585    best_similarity = max(label_similarity, content_similarity)586    587    # Normalize weights588    total_weight = weight_method + weight_similarity589    if total_weight <= 0:590        return original_confidence591        592    w_m = weight_method / total_weight593    w_s = weight_similarity / total_weight594    595    final_score = (w_m * original_confidence) + (w_s * best_similarity)596    597    return max(0.0, min(1.0, final_score))

2083        # Find matching entities using hybrid similarity; fall back to a2084        # synthetic entity so multi-value results are never silently dropped.2085        subject_entity = match_entity(subject_text, entities)2086        object_entity = match_entity(object_text, entities)20872088        if not subject_entity:2089            subject_entity = Entity(2090                text=subject_text, label="UNKNOWN",2091                start_char=0, end_char=len(subject_text),2092                confidence=0.8, metadata={"synthetic": True},2093            )2094        if not object_entity:2095            object_entity = Entity(2096                text=object_text, label="UNKNOWN",2097                start_char=0, end_char=len(object_text),2098                confidence=0.8, metadata={"synthetic": True},2099            )

1040        prompt = f"""Extract named entities from the provided text.1041Return the result as a JSON object with an "entities" key containing the list of entities.1042Each entity should have 'text', 'label', and 'confidence' fields.10431044IMPORTANT: 1045- Return a FLAT LIST of entities. 1046- DO NOT group entities by type.1047- The output structure must exactly match: {{ "entities": [ {{ "text": "...", "label": "...", "confidence": ... }}, ... ] }}10481049Example output (JSON format only):1050{{1051  "entities": [1052    {{"text": "Entity Name", "label": "CATEGORY", "confidence": 0.95}},1053    {{"text": "Another Entity", "label": "OTHER_CATEGORY", "confidence": 0.90}}1054  ]1055}}10561057Instructions:10581. Extract entities ONLY from the text provided below.10592. Do not include any entities from the example above.10603. {entity_types_instruction}10611062Text to extract from:1063{text}"""

1879TEMPORAL EXTRACTION RULES:1880- valid_from: ISO 8601 date or exact phrase from the text for when this relation became valid. Set to null if no temporal signal is present.1881- valid_until: ISO 8601 date or exact phrase for when this relation ceased. Set to null if open-ended or absent.1882- temporal_confidence (float 0.0–1.0) — calibrated as follows:1883    1.00 = full ISO date ("2022-03-15", "March 15, 2022")1884    0.90 = explicit year + month ("March 2022", "2022-03")1885    0.85 = explicit year only ("in 2022", "since 2021", "from 2019")1886    0.75 = quarter ("Q3 2023", "Q2 2021")1887    0.65 = named season or approximate range ("summer 2022", "early 2020s", "mid-2022")1888    0.50 = vague relative with computable anchor ("last year", "three months ago")1889    0.35 = highly vague relative ("recently", "years ago", "in the past")1890    0.00 = no temporal signal present for this relation1891- temporal_source_text: the EXACT verbatim substring from the source text that contains the temporal signal. Set to null when temporal_confidence is 0.0.18921893IMPORTANT: Do NOT invent or guess dates. If the text contains no temporal signal for a relation, set valid_from and valid_until to null and temporal_confidence to 0.0.

7`semantica.semantic_extract` extracts structured information from unstructured text: the foundation of every knowledge graph in Semantica:89- `NERExtractor`: named entity recognition with confidence scores and source attribution10- `RelationExtractor`: typed relationship extraction (`founded_by`, `located_in`, and custom types)11- `TripletExtractor`: direct `(subject, predicate, object)` triplet generation for RDF output12- `EventDetector`: event detection with participants, temporal context, and confidence13- Three extraction modes on every extractor: `"pattern"` (no API key), `"huggingface"`, `"llm"`

638**Treating extraction as guaranteed truth.** Semantic extraction produces confidence scores for a reason — even high-confidence extractions can be incorrect. Always validate critical extractions, especially for high-stakes decisions in security, clinical, or financial contexts.639640**Ignoring confidence thresholds.** Low-confidence extractions often indicate ambiguous text, poor model fit, or noisy input. Setting appropriate thresholds (typically 0.65-0.85) filters unreliable results before they pollute downstream processing.641642**Skipping entity resolution.** Different mentions of the same entity ("NATO", "North Atlantic Treaty Organization", "the alliance") will create duplicate nodes in your knowledge graph. Always run coreference resolution and entity deduplication.643644**Poor OCR or poor input quality.** Semantic extraction depends on readable text. Documents with OCR errors, encoding issues, or heavy redaction will produce unreliable extractions. Clean and validate input text before extraction.645646**Using LLM extraction where regex is sufficient.** For highly structured patterns like CVE identifiers (CVE-YYYY-NNNN), IP addresses, email addresses, or UUIDs, regular expressions are faster, cheaper, and more reliable than semantic extraction.

85    async def _process_document(self, text: str, entity_types: list[str]) -> str:86        messages_builder = CompletionMessagesBuilder().add_user_message(87            self._extraction_prompt.format(**{88                INPUT_TEXT_KEY: text,89                ENTITY_TYPES_KEY: ",".join(entity_types),90            })91        )9293        response: LLMCompletionResponse = await self._model.completion_async(94            messages=messages_builder.build(),95        )  # type: ignore96        results = response.content97        messages_builder.add_assistant_message(results)9899        # if gleanings are specified, enter a loop to extract more entities100        # there are two exit criteria: (a) we hit the configured max, (b) the model says there are no more entities101        if self._max_gleanings > 0:102            for i in range(self._max_gleanings):103                messages_builder.add_user_message(CONTINUE_PROMPT)104                response: LLMCompletionResponse = await self._model.completion_async(105                    messages=messages_builder.build(),106                )  # type: ignore107                response_text = response.content108                messages_builder.add_assistant_message(response_text)109                results += response_text110111                # if this is the final glean, don't bother updating the continuation flag112                if i >= self._max_gleanings - 1:113                    break114115                messages_builder.add_user_message(LOOP_PROMPT)116                response: LLMCompletionResponse = await self._model.completion_async(117                    messages=messages_builder.build(),118                )  # type: ignore119                if response.content != "Y":120                    break121122        return results

128CONTINUE_PROMPT = "MANY entities and relationships were missed in the last extraction. Remember to ONLY emit entities that match any of the previously extracted types. Add them below using the same format:\n"129LOOP_PROMPT = "It appears some entities and relationships may have still been missed. Answer Y if there are still entities or relationships that need to be added, or N if there are none. Please answer with a single letter Y or N.\n"

124    def _process_result(125        self,126        result: str,127        source_id: str,128        tuple_delimiter: str,129        record_delimiter: str,130    ) -> tuple[pd.DataFrame, pd.DataFrame]:131        """Parse the result string into entity and relationship data frames."""132        entities: list[dict[str, Any]] = []133        relationships: list[dict[str, Any]] = []134135        records = [r.strip() for r in result.split(record_delimiter)]136137        for raw_record in records:138            record = re.sub(r"^\(|\)$", "", raw_record.strip())139            if not record or record == COMPLETION_DELIMITER:140                continue141142            record_attributes = record.split(tuple_delimiter)143            record_type = record_attributes[0]144145            if record_type == '"entity"' and len(record_attributes) >= 4:146                entity_name = clean_str(record_attributes[1].upper())147                entity_type = clean_str(record_attributes[2].upper())148                entity_description = clean_str(record_attributes[3])149                entities.append({150                    "title": entity_name,151                    "type": entity_type,152                    "description": entity_description,153                    "source_id": source_id,154                })155156            if record_type == '"relationship"' and len(record_attributes) >= 5:157                source = clean_str(record_attributes[1].upper())158                target = clean_str(record_attributes[2].upper())159                edge_description = clean_str(record_attributes[3])160                try:161                    weight = float(record_attributes[-1])162                except ValueError:163                    weight = 1.0164165                relationships.append({166                    "source": source,167                    "target": target,168                    "description": edge_description,169                    "source_id": source_id,170                    "weight": weight,171                })172173        entities_df = pd.DataFrame(entities) if entities else _empty_entities_df()174        relationships_df = (175            pd.DataFrame(relationships) if relationships else _empty_relationships_df()176        )177178        return entities_df, relationships_df