
    exi?                     D   d dl mZmZmZ d dlmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ d dlmZ d d	lmZ d
edefdZdd
edededee   fdZ	 	 ddededee   dededeeef   fdZ	 	 ddedededededeeef   fdZ	 ddedededeeef   fdZy)    )DictAnyList)SessionN)get_pdf_parser)logger)settings)BnkParsedDocumentBnkDocumentChunkBnkConsolidatedLLMOutputBnkAIReqRes)AzureOpenAIClient)!build_response_from_openai_resulttextreturnc                     t        |       dz  S )u1   Estimate token count (rough: 1 token ≈ 4 chars)   )len)r   s    D/var/www/html/BankruptcyAI-uat/bankruptcy-ai/src/utils/file_chunk.pyestimate_tokensr      s    t9>    
max_tokensoverlap_tokensc                 >   t        |       }||k  rd| |dgS g }|dz  }|dz  }d}d}|t        |       k  rt        ||z   t        |             }	|	t        |       k  r<dD ]7  }
| ||	 j                  |
      }|dk7  s||dz  kD  s&||z   t        |
      z   }	 n | ||	 j	                         }|j                  ||t        |      d       |	t        |       k\  rn |	|z
  }|dk  rd}|dz  }|t        |       k  rt        j                  dt        |       d	| d
       |S )a=  
    Split text into chunks with overlapping for context continuity
    
    Args:
        text: Full extracted text
        max_tokens: Max tokens per chunk
        overlap_tokens: Number of tokens to overlap between chunks
    
    Returns:
        List of chunk dicts with chunk_index, chunk_text, token_count
    r   )chunk_index
chunk_texttoken_countr   )z. z.
z!
z?
z

      z[Chunk] Created z chunks with z token overlap)r   r   minrfindstripappendr   info)r   r   r   total_tokenschunks	max_charsoverlap_charsstart	chunk_idxend	separatorlast_sepr   s                r   chunk_text_with_overlapr/      sh    #4(Lz!'
  	 FQI"Q&MEI
#d)
%)#SY/ T?@ 	c?00;r>ha&?(*S^;C	 %_**,
$$*:6
 	 #d)m#19EQ	1 #d)
4 KK"3v;-}^<LN[\Mr   dbai_req_res_id	file_urlsc           
        K   g }g }|D ]  }	 t               }|j                  |      }	|j                  |	      }
t        j                  dt        |
       d|        |
st        j                  d|        pt        |
||      }t        |||
t        |
      t        |            }| j                  |       | j                          | j                  |       |j                  |j                          t        j                  d|j                           |D ]q  }t#        |j                   |d	   |d
   |d         }| j                  |       | j                          | j                  |       |j                  |j                          s  d|||t        |      t        |      dS # t        $ r&}t        j                  d| d|        Y d}~d}~ww xY ww)a  
    Task A: Extract text from documents and store chunks in DB
    Works for 1 or many documents
    
    Args:
        db: Database session
        ai_req_res_id: Primary key ID from bnk_ai_req_res table
        file_urls: List of document URLs (can be single item)
        max_tokens: Max tokens per chunk
        overlap_tokens: Overlap between chunks
    
    Returns:
        Dict with all parsed_document_ids and chunk_ids
    z[Task A] Extracted z chars from z[Task A] Extraction failed for z: Nz [Task A] No text extracted from )r1   
source_urlextracted_textr   chunkedz$[Task A] Saved BnkParsedDocument id=r   r   r   )parsed_document_idr   r   r   T)successr1   parsed_document_ids	chunk_idsdocuments_processedtotal_chunks)r   	parse_urlget_textr   r%   r   	Exceptionerrorwarningr/   r
   r   addcommitrefreshr$   idr   )r0   r1   r2   r   r   all_parsed_docsall_chunk_idsfile_urlparser	documentsr5   er'   
parsed_doc
chunk_datachunk_records                   r   task_extract_and_chunkrO   Q   s    * OM +2	#%F((2I#__Y7NKK-c..A-B,xjYZ
 NN=hZHI )^T '')'7K

 	z
		


:z}}-::==/JK ! 
	2J+#-==&}5%l3&}5	L FF< IIKJJ|$  1
	2C+2\ &.""?3M* M  	LL:8*BqcJK	s0   GAF+EG+	G4GGGG	file_typedata
request_idc                   K   | j                  t              j                  t        j                  |k(        j	                         }|sdd| dS t               }d}d}|D ]  }	| j                  t              j                  t        j                  |	j                  k(  t        j                  dk(        j                  t        j                        j	                         }
d}|
D ]i  }	 t        j                  d|j                   d| d       |j                  ||xs i |	j                  xs d	|j                   |
       d{   }t        j                  d|j                   dt#        |       dt%        |t&              rd|v nd        t%        |t&              rr|j)                  d      ra|j)                  d      dk(  rMt        j*                  d|j                   d|	j                   d       ||_	        | j-                          d}d} nA||_	        | j-                          |dz  }t        j                  d|j                          l |st        j                  d|	j                   d       | j                  t              j                  t        j                  |	j                  k(  t        j                  dk(        j	                         }|D ]  }dddd|_	         | j-                           d|||dS 7 # t.        $ rR}t        j0                  d|j                   d|        dt3        |      i|_	        | j-                          Y d}~xd}~ww xY ww)a  
    Task B: Read unprocessed chunks from DB and send to LLM
    Uses AzureOpenAIClient.verify_document with pre-extracted chunk text
    
    Args:
        db: Database session
        ai_req_res_id: Primary key ID from bnk_ai_req_res table
        file_type: Document type for prompt selection (same as original flow)
        data: Payload data from request (for verification matching)
        request_id: Request ID for Pusher notifications (jailbreak errors only)
    
    Returns:
        Dict with processed chunk count
    F&No parsed documents for ai_req_res_id=r8   r@   r   Nz[Task B] Sending chunk id=z to LLM with file_type='z&' (prompt selected based on this type) )rP   payloadrH   r5   rR   z [Task B] Got response for chunk z: type=z, has_error=r@   zN/A
error_typecontent_filterz&[Task B] Jailbreak detected for chunk z of document z@ - notification sent, skipping remaining chunks of this documentTr    z([Task B] Stored llm_output for chunk id=z[Task B] Chunk id=z failed with exception: z.[Task B] Skipped remaining chunks of document z due to jailbreakz;Skipped due to jailbreak in previous chunk of same document)r@   rX   message)r8   r1   chunks_processedjailbreak_detected)queryr
   filterr1   allr   r   r7   rE   
llm_outputorder_byr   r   r%   verify_documentr4   r   type
isinstancedictgetrA   rC   r?   r@   str)r0   r1   rP   rQ   rR   parsed_docsclientprocessed_countr\   rL   r'   document_has_jailbreakchunkr`   rK   remaining_chunksremaining_chunks                    r   task_process_chunks_llmro      s    , ((,-44''=8	ce   -STaSb+cdd  FO! 9
*+22//:==@''4/
 (#//
0 	
 "' !	E 8
BZ[dZe  fL  M  N $*#9#9' JB'228b#(#3#3) $: $ 
 >uxxjPTU_P`Oaam  HR  S]  _c  Hdnu  zD  oD  jo  np  q  r j$/JNN74KPZP^P^_kPl  qA  QANN%KEHH:Ubcmcpcpbq  rr  $s  t'1E$IIK-1*)-&#- 		1$FuxxjQR9!	F "KKHWhij!xx(89@@ 33z}}D ++t3  ce  $4 !"2\.* IIKs9~ &+0	 _0  1%((;STUSVWX$+SV#4 		sX   C&M$)ALLC	LM$>LM$B.M$L	M!AMM$M!!M$c                 
  K   | j                  t              j                  t        j                  |k(        j	                         }|sdd| dS g }|D ]W  }| j                  t
              j                  t
        j                  |j                  k(        j                  t
        j                        j	                         }t        j                  dt        |       d|j                          |D ]  }t        j                  d|j                   d|j                  du d	t        |j                                |j                  sXt        j                  d|j                   d
t        |j                  t               r|j                  j#                         nd        t        |j                  t               r|j                  j%                  dd      }|j                  j%                  d      }	t        j                  d|j                   d| d|	        |r*|	dk7  r%t        j                  d|j                   d       Vt        j                  d|j                   d       |j'                  |j                  |j                  |j(                  |j                  d        Z t        j                  dt        |              |sdddS t        |      dk(  r5|d   d   }
|d   j%                  dd      g}t        j                  d       n\|d   d   }g }t+               }|D ]  }|j%                  di       }t        |t               s&|j%                  dg       }|D ]|  }|j%                  dd      }||vr#|j'                  |       |j-                  |       <|j%                  d d      sOt/        |      D ]   \  }}|j%                  d      |k(  s|||<    | ~  t        |t               r|j1                         ni }
||
d<   |rt	        d! |D              nd}||
d"<   t3        t+        d# |D                    }t        j                  d$t        |       d%t        |       d&       |rd'j5                  |      nd}t7        |
||      }t9        |d(      r|j!                         n|}t;        ||)      }| j-                  |       | j=                          t        j                  d*       | j                  t>              j                  t>        j                  |k(        jA                         }|r,||_!        | j=                          t        j                  d+       d,|t        |      |d-S w).a&  
    Task C: Consolidate all chunk LLM outputs into final response
    
    Args:
        db: Database session
        ai_req_res_id: Primary key ID from bnk_ai_req_res table
        file_type: Document type for response formatting
    
    Returns:
        Dict with consolidated response
    FrT   rU   z[Task C] Found z chunks for parsed_doc_id=z[Task C] Chunk z: llm_output exists=Nz, type=z output keys: znot dictr@   rX   z: has_error=z, error_type=rY   z[Task C] Skipping chunk z - non-jailbreak errorz[Task C] Including chunk z in outputs)chunk_idr   r4   outputz"[Task C] Total outputs collected: z#No valid LLM outputs to consolidater    r   rr   r4   rV   z/[Task C] Single chunk - no consolidation neededanalysis_results
field_nameis_matchc              3   @   K   | ]  }|j                  d d        yw)ru   FNrf   ).0rs     r   	<genexpr>z+task_consolidate_outputs.<locals>.<genexpr>r  s     Tq!%%
E2Ts   overall_matchc              3   d   K   | ](  }|j                  d       s|j                  d d       * yw)r4   rV   Nrw   )rx   cs     r   rz   z+task_consolidate_outputs.<locals>.<genexpr>v  s'     c1quuUaObquu\26cs   00z[Task C] Merged z chunk outputs, z analysis fields,re   )r1   consolidated_responsez'[Task C] Saved BnkConsolidatedLLMOutputz%[Task C] Updated BnkAIReqRes.responseT)r8   r1   chunk_countr   )"r]   r
   r^   r1   r_   r   r7   rE   ra   r   r   r%   r   r`   rc   rd   re   keysrf   r$   r4   setrB   	enumeratecopylistjoinr   hasattrr   rC   r   firstresponse)r0   r1   rP   rh   all_outputsrL   r'   rl   	has_errorrX   raw_responsesource_urlsbase_outputmerged_analysis_resultsseen_fieldsrM   chunk_outputrs   resultrt   iexistingall_matchedrH   formatted_responser   consolidated_recordjobs                               r   task_consolidate_outputsr   
  s@    " ((,-44''=8	ce   -STaSb+cddK! !
*+22//:==@

(#//
0 	 	oc&k]2LZ]]O\] 	EKK/%((3GHXHX`dHdGeelmqrw  sC  sC  nD  mE  F  GoehhZ~aklql|l|  C  bDeFVFVF[F[F]  JT  FU  V  W e..5 % 0 0 4 4We DI!&!1!1!5!5l!CJKK/%((<	{R_`j_k lm !Z3C%C&>uxxjH^$_` 7zMN"" %#(#4#4","7"7#..	$ +	!F KK4S5E4FGH +PQQ ;1"1~h/"1~)),;<EF "!nX. #%e% 	*J%>>(B7L,-#/#3#34F#K . *F!'L"!=J!4/66v>#
3 "::j%8/89P/Q *8#+<<#=#KAG$;A$>$)**		*( .8T-J{'')PR+B'( YpcT<STTuz(3_% 3cccd&s;'7&88HMdIeHffvwx )4sxx$H:<T\] :AASU[9\.335bt 3#3 FFIIK
KK9; ((;

&
&{~~'F
G
M
M
OC
,
		;= &;'!6	 s!   EU:
HU:A"U:9%U:FU:)i i  )NN)unknown)typingr   r   r   sqlalchemy.ormr   aiohttpjsonsrc.utils.pdf_parserr   src.core.loggingr   src.core.configr	   src.modules.modelsr
   r   r   r   src.utils.azure_openair   src.modules.prompt.controllerr   rg   intr   r/   rO   ro   r    r   r   <module>r      sO   " " "   / # $ i i 4 K# # 
7# 73 7QT 7_cdh_i 7D LLL CyL 	L
 L 
#s(^Lh eee e 	e
 e 
#s(^eX LLL L 
#s(^	Lr   