
    Ԕ9i?                     X    d dl Z d dlZd dlZ	 d Zd Zd Zd Zd Zd Z	d Z
dd	Zdd
Zy)    Nc                     t        j                  | d      }t        j                  dk(  r|j	                  d       |d   }|j                  d|dz         d d dd f   j                         S )	Nint32dtypebigTinplacer      )npfromfilesys	byteorderbyteswapreshapecopyfnameads      _/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/faiss/contrib/vecs_io.py
ivecs_readr      s^    
E)A
}}	

4
 	!A99RQ12&++--    c                 6    t        |       j                  d      S Nfloat32)r   viewr   s    r   
fvecs_readr          e!!),,r   c                     t         j                  dk7  sJ t        j                  | dd      }|d   }|j	                  d|dz         d d dd f   S )Nr   r   rr   moder   r
   r   )r   r   r   memmapr   r   s      r   
ivecs_mmapr&      sO    ==E!!!
		%wS1A	!A99RQ12&&r   c                 6    t        |       j                  d      S r   )r&   r   r   s    r   
fvecs_mmapr(   #   r    r   c                    t        j                  | dd      }t        j                  dk(  r.|d d d d d   j	                         }|j                  d      d   }n|d d j                  d      d   }|j                  d|dz         d d dd f   S )	Nuint8r"   r#   r      r
   r   r   )r   r%   r   r   r   r   r   )r   xdar   s       r   
bvecs_mmapr.   '   s    
		%wS1A
}}rU4R4[GGGQbqEJJw"99RQ12&&r   c                     |j                   \  }}t        j                  ||dz   fd      }||d d df<   ||d d dd f<   t        j                  dk(  r|j                  d       |j                  |        y )Nr   r   r   r   r   Tr   )shaper   emptyr   r   r   tofile)r   mnr   m1s        r   ivecs_writer6   1   si    77DAq	1a!e*G	,BBq!tHBq!"uI
}}
D!IIer   c                 \    |j                  d      }t        | |j                  d             y )Nr   r   )astyper6   r   )r   r3   s     r   fvecs_writer9   ;   s"    	Aqvvg'r   c              #     K   t         j                  j                  |       }t        | d      5 }t	        j
                  |j                  d      d      d   }ddd       dz   }||z  }t	        j                  | dt        j                        }|j                  ||      }t        d||      D ])  }	t	        j                  |	|z   |g      }
||	|
ddf    + y# 1 sw Y   xY ww)	zG
    Memory-mapped iterator - only loads requested slices into RAM
    rbr+   <i4r   r   Nr"   r$   r   )ospathgetsizeopenr   
frombufferreadr%   r*   r   rangemin)filepath
batch_size	file_sizefdimbytes_per_vec	n_vectorsmmrecordsstartends              r   
bvecs_iterrQ   ?   s     
 )I	h	 7mmAFF1IU3A67 GM]*I	8#RXX	6BjjM2Gq)Z0 %ffej()45eCim$$%7 7s   ,C(*CBC(C%!C(c           
   #      K   g }t        j                  |       D ]f  }|j                         s|j                  j	                  d      s0|j                  j                  d      sL|j                  |j                         h |j                          |st        d|        g }|D ]b  }t         j                  j                  |      }	 |j                  d      d   j                  d      d   }|j                  t        |             d t        t        t!        |                  }t#        |      |k7  rIt%        |      t%        |      z
  }	t        d	t#        |	       d
t#        |       dt!        |      dz
         t'        |d   d      5 }
t)        j*                  |
j-                  d      d      d   }ddd       dz   }d}d}|D ]  }t         j                  j/                  |      }||z  }t)        j0                  |dt(        j2                        }|j5                  ||      }|ddddf   }d}|L||z
  }||k  r%t)        j6                  ||d| g      }| d}d}|}nt)        j6                  ||g      }||z  }||z   |k  r||||z     ||z  }||z   |k  r||z
  }|dkD  s||d j9                         }|} || yy# t        t        f$ r t        d|       w xY w# 1 sw Y   :xY ww)a  
    Memory-mapped iterator over chunked .bvecs files.
    Iterates through all chunk files in order (chunk_0000.bvecs, chunk_0001.bvecs, etc.)
    and yields batches of vectors, handling cases where batches span multiple files.

    Args:
        chunk_folder: path to folder containing chunk_XXXX.bvecs files
        batch_size: number of vectors to yield per batch

    Yields:
        numpy array of shape (batch_size, d) or smaller for last batch

    Raises:
        ValueError: if there are gaps in the chunk sequence
    chunk_z.bvecszNo chunk files found in _r   .r   zInvalid chunk filename format: z0Gap detected in chunk sequence! Missing chunks: z
Found chunks: z(
Expected continuous sequence from 0 to r;   r+   r<   r   Nr"   r=   )r>   scandiris_filename
startswithendswithappendr?   sort
ValueErrorbasenamesplitint
IndexErrorlistrD   lensortedsetrA   r   rB   rC   r@   r%   r*   r   vstackr   )chunk_folderrG   chunk_filesentrychunk_numbersr?   r^   num_strexpected_chunksmissingrI   rJ   rK   bufferbuffer_size
chunk_pathrH   rL   rM   rN   vectorsrO   neededbatch	remainders                            r   bvecs_iter_chunkedru   R   s=    $ KL) +==?uzz44X>5::CVCVW_C`uzz*+ 3L>BCC M K77##D)	KnnS)!,2237:G  W.	K 5]!345Om/o&]);;>vg>O P#M23 4669-6H6J5KM
 	
 
k!nd	# 7qmmAFF1IU3A67 GM FK " #$
GGOOJ/	.	 YYz288<**Y6!QR%. +-F"		677F+;"<= FG#45y( j I-%
 233ZE j I- %	q=UV_))+F#KG#$J  { J' 	K>xjIJJ	K7 7sV   +K>K>
K>&A#K>
A K
BK>*K19C,K>&
K>1K>K..K>1K;6K>)i )r   numpyr   r>   r   r   r&   r(   r.   r6   r9   rQ   ru    r   r   <module>rx      sC      	.-'-'(%&`r   