
    ߔ9iE                        d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( erddl)m*Z*  e$jV                  e,      Z- ej\                  d      Z/dZ0dddZ1e G d d             Z2e%	 d:dee2   dee3   de3de3dee3   deee4   ee4   f   fd       Z5 G d de      Z6 G d d e      Z7d!d"d#edee3   ddfd$Z8d%e4fd&Z9d'e4fd(Z:d)e4fd*Z;d!d"d+e3ddfd,Z<d!d"d-ed.e=d+e3ddf
d/Z>d-ed0e2d.e=dee3   fd1Z?d2ee   d3e3de7fd4Z@d!d"d5ee3   d.e=dee   fd6ZAd!d"d5ee3   d.e=dee   fd7ZB G d8 d9e      ZCy);z.Git LFS related type definitions and utilities    N)AbstractContextManager)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)HTTPBasicAuth)ENDPOINTHF_HUB_ENABLE_HF_TRANSFERREPO_TYPES_URL_PREFIXES)get_session   )get_token_to_sendhf_raise_for_statushttp_backoffloggingvalidate_hf_hub_args)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                   r    e Zd ZU dZeed<   eed<   eed<   edefd       Z	edefd       Z
ed	efd
       Zy)
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesamplepathc                     t        |      }t        j                  |d      5 }|j                  d      d d }t	        |      }d d d         | |      S # 1 sw Y   xY w)Nrb   r!   r   r"   )r   ioopenpeekr   )clsr#   r!   filer"   shas         ]/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/huggingface_hub/lfs.py	from_pathzUploadInfo.from_pathF   s_    t}WWT4  	$DYYs^DS)Fd#C	$ S88	$ 	$s    AAdatac                 b    t        |      j                         } | t        |      |d d |      S )Nr&   )r!   r"   r   )r   digestlen)r+   r0   r-   s      r.   
from_byteszUploadInfo.from_bytesN   s-    Tl!!#D	$t*SAA    fileobjc                     |j                  d      }|j                  dt        j                         t	        |      }|j                         }|j                  dt        j                          | |||      S )Nr&   r   r'   )readseekr(   SEEK_SETr   tell)r+   r6   r"   r-   r!   s        r.   from_fileobjzUploadInfo.from_fileobjS   sX    c"Q$'"||~Q$S88r5   N)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr/   r4   r	   r<    r5   r.   r    r    3   sj     M
IM9S 9 9 Be B B 98 9 9r5   r    upload_infostoken	repo_typerepo_idendpointreturnc                 Z   ||nt         }d}|t        v r	t        |   }| d| | d}t               j                  |t        dddg| D cg c])  }|j
                  j                         |j                  d+ c}d	d
t        dt        |xs d                  }t        |       |j                         }	|	j                  dd      }
t        |
t              st        d      |
D cg c]  }d|vst!        |       c}|
D cg c]  }d|v st#        |       c}fS c c}w c c}w c c}w )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`, *optional*):
            An authentication token ( See https://huggingface.co/settings/tokens )

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        `ValueError`: If an argument is invalid or the server response is malformed

        `HTTPError`: If the server returned an error
    N /z.git/info/lfs/objects/batchuploadbasic	multipartoidr!   r   )	operation	transfersobjects	hash_algoaccess_tokenT)headersjsonauthrW   zMalformed response from servererror)r   r   r   postLFS_HEADERSr   hexr!   r   r   r   r[   get
isinstancelist
ValueError_validate_batch_actions_validate_batch_error)rG   rH   rI   rJ   rK   
url_prefix	batch_urlrP   resp
batch_inforW   objs               r.   post_lfs_batch_inforl   ]   sH   F $/xXHJ++,Y7
*Aj\'2MNI=!!;/ +
  "==,,."KK "
 emt,
  D( JnnY-Ggt$9:: 29O#G3<N	 	%O/6I'S.	s	#I +, 	PIs   	.D&	D#0D#	D(D(c                   "    e Zd ZU eed<   eed<   y)PayloadPartT
partNumberetagN)r=   r>   r?   rC   rB   rE   rF   r5   r.   rn   rn      s    O
Ir5   rn   c                   ,    e Zd ZU dZeed<   ee   ed<   y)CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rT   partsN)r=   r>   r?   r@   rE   rB   r   rn   rF   r5   r.   rr   rr      s    I	Hr5   rr   rU   r   lfs_batch_actionc           
      0   t        |       |j                  d      }|$t        j                  d| j                   d       y|d   d   }t        |       |d   j                  d      }|t        |       |j                  di       }|j                  d      }|	 t        |      }t        | |||d          nt        | |d          |t        |       t               j                  |d   t        dt        |xs d            | j                  j                   j#                         | j                  j$                  d      }t'        |       t        j                  | j                   d       y# t        t        f$ r t        d	| d
      w xY w)a  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        token (`str`, *optional*):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises:
        - `ValueError` if `lfs_batch_action` is improperly formatted
        - `HTTPError` if the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrP   verifyheader
chunk_sizezTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.href)rU   rx   ry   
upload_url)rU   r{   USERT)usernamepasswordrS   )r\   r[   z: Upload successful)re   ra   loggerdebugpath_in_repo_validate_lfs_actionrC   rd   	TypeError_upload_multi_part_upload_single_partr   r^   r   r   upload_infor   r`   r!   r   )	rU   rt   rH   rv   upload_actionverify_actionrx   ry   verify_resps	            r.   
lfs_uploadr      s   ( ,-""9-G'	(>(>'??mno %Y/9M'$Y/33H=M ]+ x,FL)J	ZJ
 	Yv*anouavwiM&<QR  ]+!m((&!9J5=TX9YZ"..5599;YEZEZE_E_` ) 

 	K(
LLI**++>?@# I& 	fgqfrrtu 	s   E6 6F
lfs_actionc                     t        | j                  d      t              r0| j                  d      *t        | j                  d      t              st	        d      | S ).validates response from the LFS batch endpointrz   rx   z"lfs_action is improperly formatted)rb   ra   rE   dictrd   )r   s    r.   r   r      sK     	:>>&)3/^^H%-JNN8<TVZ1[=>>r5   lfs_batch_actionsc                 P   t        | j                  d      t              rt        | j                  d      t              st	        d      | j                  di       j                  d      }| j                  di       j                  d      }|t        |       |t        |       | S )r   rT   r!   z)lfs_batch_actions is improperly formattedrv   rP   rw   )rb   ra   rE   rC   rd   r   )r   r   r   s      r.   re   re      s    (,,U3S9jIZI^I^_eIfhk>lDEE%)))R8<<XFM%)))R8<<XFM ]+ ]+r5   lfs_batch_errorc                 l   t        | j                  d      t              rt        | j                  d      t              st	        d      | j                  d      }t        |t
              r>t        |j                  d      t              rt        |j                  d      t              st	        d      | S )r   rT   r!   z'lfs_batch_error is improperly formattedr]   messagecode)rb   ra   rE   rC   rd   r   )r   
error_infos     r.   rf   rf     s    **5137JGZGZ[aGbdg<hBCC $$W-J:t$z~~i0#6z~~f-s3BCCr5   r{   c                     | j                  d      5 }t        d||      }t        |       ddd       y# 1 sw Y   yxY w)aZ  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises: `requests.HTTPError` if the upload resulted in an error
    T	with_tqdmPUTr0   N)as_filer   r   )rU   r{   r6   responses       r.   r   r     sB     
		T		* &gz@H%& & &s   6?rx   ry   c                    t        || j                  |      }t        }t        rKt        | j                  t
              s1t        | j                  t              st        j                  d       d}|rt        | ||      nt        | ||      }t               j                  |t        || j                  j                  j                               t               }t#        |       y)z@
    Uploads file using HF multipart LFS transfer protocol.
    )rx   r   ry   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rU   sorted_parts_urlsry   )r[   rZ   N)_get_sorted_parts_urlsr   r   rb   path_or_fileobjrE   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   r^   _get_completion_payloadr   r`   r_   r   )rU   rx   ry   r{   r   use_hf_transferresponse_headerscompletion_ress           r.   r   r   '  s    
 /f)J_J_lvw 0O!944c:944d;	
    	"IIZgqr&N_lvw  !]''$%5y7L7L7S7S7W7W7YZ ( N
 'r5   r   c           
      V   t        | j                         D cg c]1  \  }}|j                         rt        |      dkD  rt	        |d      |f3 c}}d       D cg c]  \  }}|	 }}}t        |      }|t        |j                  |z        k7  rt        d      |S c c}}w c c}}w )Nr   
   c                     | d   S Nr   rF   )ts    r.   <lambda>z(_get_sorted_parts_urls.<locals>.<lambda>S  s
    !A$ r5   )keyz0Invalid server response to upload large LFS file)sorteditemsisdigitr3   rC   r   r!   rd   )rx   r   ry   part_numr{   _sorted_part_upload_urls	num_partss           r.   r   r   J  s     $ -3LLN(Hj##%#h-!*; Xr"J/
 

Az 	
 
 +,ID))J677KLL""
s   6B
B%r   rT   c                     g }t        |       D ]H  \  }}|j                  d      }||dk(  rt        d| d|dz          |j                  |dz   |d       J ||dS )Nrp   rN   zInvalid etag (`z`) returned for part r   )ro   rp   )rT   rs   )	enumeratera   rd   append)r   rT   rs   part_numberrx   rp   s         r.   r   r   \  s     "E()9: 	
Vzz&!<42:tf4I+XY/IZ[\\)Ao	
		
 ''r5   r   c                 6   g }| j                  d      5 }t        |      D ]S  \  }}t        |||z  |      5 }t        d||      }t	        |       |j                  |j                         d d d        U 	 d d d        |S # 1 sw Y   jxY w# 1 sw Y   |S xY w)NTr   )	seek_from
read_limitr   r   )r   r   SliceFileObjr   r   r   rZ   )	rU   r   ry   rZ   r6   part_idxpart_upload_urlfileobj_slicepart_upload_ress	            r.   r   r   k  s     G			T		* 	8g)23D)E 	8%Ho$x/% 8 ".uoM"Z#O46678 8	8	8 N8 8	8 Ns"   #B5B-
BBBBc                     	 ddl m} 	  || j                  ||ddd      S # t        $ r t        d      w xY w# t
        $ r}t        d      |d }~ww xY w)	Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.         )	file_path
parts_urlsry   	max_filesparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.)hf_transferr   ImportErrorrd   r   	ExceptionRuntimeError)rU   r   ry   r   es        r.   r   r   |  s    
0//(!!
 	
  
Q
 	

  &
 	s   ! 9 6	AAAc                   |    e Zd ZdZdededefdZd Zd Zddefd	Z	d
efdZ
ej                  fdeded
efdZd Zy)r   a  
    Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.

    This is NOT thread safe

    Inspired by stackoverflow.com/a/29838711/593036

    Credits to @julien-c

    Args:
        fileobj (`BinaryIO`):
            A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
            `fileobj` will be reset to its original position when exiting the context manager.
        seek_from (`int`):
            The start of the slice (offset from position 0 in bytes).
        read_limit (`int`):
            The maximum number of bytes to read from the slice.

    Attributes:
        previous_position (`int`):
            The previous position

    Examples:

    Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
    ```python
    >>> with open("path/to/file", "rb") as file:
    ...     with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
    ...         fslice.read(...)
    ```

    Reading a file in chunks of 512 bytes
    ```python
    >>> import os
    >>> chunk_size = 512
    >>> file_size = os.getsize("path/to/file")
    >>> with open("path/to/file", "rb") as file:
    ...     for chunk_idx in range(ceil(file_size / chunk_size)):
    ...         with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
    ...             chunk = fslice.read(...)

    ```
    r6   r   r   c                 .    || _         || _        || _        y N)r6   r   r   )selfr6   r   r   s       r.   __init__zSliceFileObj.__init__  s    "$r5   c                 P   | j                   j                         | _        | j                   j                  dt        j
                        }t        | j                  || j                  z
        | _	        | j                   j                  | j                  t        j                         | S r   )r6   r;   _previous_positionr9   osSEEK_ENDminr   r   _lenr(   r:   )r   end_of_streams     r.   	__enter__zSliceFileObj.__enter__  sm    "&,,"3"3"5))!R[[9)GH	$.."++6r5   c                 l    | j                   j                  | j                  t        j                         y r   )r6   r9   r   r(   r:   )r   exc_type	exc_value	tracebacks       r.   __exit__zSliceFileObj.__exit__  s     $112;;?r5   nc                     | j                         }|| j                  k\  ry| j                  |z
  }| j                  j                  |dk  r|      }|S t	        ||            }|S )Nr5   r   )r;   r   r6   r8   r   )r   r   posremaining_amountr0   s        r.   r8   zSliceFileObj.read  sc    iik$))99s?||  QU!1Y ADAGW@XYr5   rL   c                 P    | j                   j                         | j                  z
  S r   )r6   r;   r   r   s    r.   r;   zSliceFileObj.tell  s    ||  "T^^33r5   offsetwhencec                    | j                   }|| j                  z   }|t        j                  t        j                  fv rD|t        j                  k(  r||z   n||z   }t        |t        ||            }t        j                  }nY|t        j                  k(  r7| j                  j                         }t        ||z
  t        |||z
              }nt        d| d      | j                  j                  ||      | j                   z
  S )Nzwhence value z is not supported)r   r   r   r:   r   maxr   SEEK_CURr6   r;   rd   r9   )r   r   r   startendcur_poss         r.   r9   zSliceFileObj.seek  s    diibkk2;;//'-'<UV^#,FFC 01F[[Fr{{"ll'')G#fcGm*DEF}VH4EFGG||  04>>AAr5   c              #   4   K   | j                  d       y w)Ni  @ )r   )r8   r   s    r.   __iter__zSliceFileObj.__iter__  s     ii/i**s   N))r=   r>   r?   r@   r	   rC   r   r   r   r8   r;   r   r:   r9   r   rF   r5   r.   r   r     sq    *X% %S %c %
@c 4c 4 /1kk B3 B Bc B+r5   r   r   )Dr@   r(   r   rer   
contextlibr   dataclassesr   mathr   os.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   requests.authr   huggingface_hub.constantsr   r   r   huggingface_hub.utilsr   utilsr   r   r   r   r   	utils.shar   r   _commit_apir   
get_loggerr=   r   compile	OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDr_   r    rE   r   rl   rn   rr   r   r   re   rf   r   rC   r   r   r   r   r   r   rF   r5   r.   <module>r     sx   5 	 	 	  - !    \ \ \ ' b b - f f * /			H	%BJJ()	5  -2 &9 &9 &9R  #E:&EC=E E 	E
 smE 4:tDz!"E EP9 
 9A. 9A$ 9AxX[} 9Aae 9AxT t 4 &#7 &S &T && ("6  (  (RU  (cf  (ko  (F#4 #j #c #VZ[^V_ #$(d4j (s (GY (#8<S	OR	$Z"#8<S	OR	$Z8W+) W+r5   