
    0Fie                    ~    d dl mZ d dlmZmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ  G d d	e          Zd
S )    )annotations)defaultdictdeque)AnyCallable)sizeof)ShardsBuffer)ResourceLimiter)
log_errorsc                  F     e Zd ZU ded<   ded<   d fdZddZddZ xZS )MemoryShardsBufferCallable[[Any], Any]_deserializezdefaultdict[str, deque[Any]]_shardsdeserializereturnNonec                    t                                          t          d                      || _        t	          t
                    | _        d S )N)memory_limiter)super__init__r
   r   r   r   r   )selfr   	__class__s     ;lib/python3.11/site-packages/distributed/shuffle/_memory.pyr   zMemoryShardsBuffer.__init__   sA    (=(=>>>'"5))    idstrshards	list[Any]c                   K   t                      5  |                     d          5  | j        |                             |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nwrite)r   timer   extend)r   r   r   s      r   _processzMemoryShardsBuffer._process   s       \\ 	0 	07## 0 0R ''///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0s4   A,!AA,A	A,A	A,,A03A0r   c           	        |                                   | j        st          d          |                     d          5  | j                            |          }| xj        t          t          t          |                    z  c_        g }|r>|                                }|
                    |                     |                     |>d d d            n# 1 swxY w Y   |S )Nz$Tried to read from file before done.read)raise_on_exception_inputs_doneRuntimeErrorr"   r   pop
bytes_readsummapr   appendr   )r   r   r   datashards        r   r&   zMemoryShardsBuffer.read   s   !!!  	GEFFFYYv 	6 	6\%%b))FOOs3vv#6#6777OO D 6

D--e44555  6	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 s    BCC C)r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   )__name__
__module____qualname____annotations__r   r$   r&   __classcell__)r   s   @r   r   r      s         &&&&))))* * * * * *
0 0 0 0       r   r   N)
__future__r   collectionsr   r   typingr   r   dask.sizeofr   distributed.shuffle._bufferr	   distributed.shuffle._limiterr
   distributed.utilsr   r    r   r   <module>r>      s    " " " " " " * * * * * * * *                       4 4 4 4 4 4 8 8 8 8 8 8 ( ( ( ( ( (         r   