o
    Nrf>                     @   s*  d dl mZ d dlZzd dlZW n ey   d dlZY nw d dlZd dl	Z
d dlZd dlZd dlZd dlmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ dd Zdd ZG dd	 d	eZG d
d deZG dd deZ G dd deZ!	dddZ"dddZ#dd Z$dS )    )print_functionN)gmtimestrftimec                 C   s&   t dt| || td d S )N    )printjoin	tracebackformat_exceptionos_exit)exc_type	exc_valueexc_traceback r   [/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/clodius/save_tiles.pyhandle_exception   s   r   c              	   C   s   t  t jt j |  r|jsEz| jdd\}}}|||| W n ttfy0   t	d Y n t
jy=   |  Y nw |  r|jr|  d S )Nr   )timeoutz
Exiting...)signalSIGINTSIG_IGNemptyvaluegetsave_binned_tileKeyboardInterrupt
SystemExitr   queueEmptyflush)qZ
tile_saverfinished
zoom_levelZtile_pos	tile_binsr   r   r   tile_saver_worker    s   r$   c                   @   sR   e Zd Z		dddZdd Zdd Zd	d
 Zdd Zdd Zdd Z	dd Z
dS )	TileSaverF        c                 C   s"   || _ || _|| _|| _|| _d S N)max_data_in_sparsebins_per_dimensionnum_dimensionsprint_statusinitial_value)selfr(   r)   r*   r+   r,   r   r   r   __init__4   s   zTileSaver.__init__c                 C      d S r'   r   )r-   tiler   r   r   	save_tileF      zTileSaver.save_tilec                 C   s0   d |dtt|}||d}| | d S )Nz{}.{}.tile_id
tile_value)formatr   mapstrr1   )r-   r"   tile_position	tile_datar5   r0   r   r   r   make_and_save_tileI   s   
zTileSaver.make_and_save_tilec           
   	      s    j g j j  }| D ]\}}tt fddt|D }	|||	< qt j dkr? ||dd |D ||d d S  ||t	t
jdd |D ||d d S )Nc                    s   g | ]\}}| j |  qS r   )r)   ).0ibpr-   r   r   
<listcomp>]   s    z-TileSaver.save_dense_tile.<locals>.<listcomp>r   c                 S   s   g | ]	}t |d  dqS )r      roundr=   vr   r   r   rA   f       Zdense	min_value	max_valuec                 S   s   g | ]	}d d |D qS )c                 S      g | ]}t |d qS rB   rC   )r=   yr   r   r   rA   r       z8TileSaver.save_dense_tile.<locals>.<listcomp>.<listcomp>r   rE   r   r   r   rA   r   rG   )r,   r)   r*   itemsintsum	enumeratelenr<   listitchainfrom_iterable)
r-   r"   r:   r#   rI   rJ   Zinitial_valuesbin_posvalindexr   r@   r   save_dense_tileT   s:   


zTileSaver.save_dense_tilec           	      C   sv   g }|  D ]'\}}t|dkr|ttt||d gg7 }q|ttt|t|gg7 }q| |||||d d S )Nr   r   )sparserI   rJ   )rO   rS   rT   r8   floatr<   )	r-   r"   r:   r#   rI   rJ   ZshownrX   Zbin_valr   r   r   save_sparse_tilez   s   
zTileSaver.save_sparse_tilec                 C   s4   t |}t|}| ||dd |D ||d dS )aQ  
        Save a tile that has all of its data in one long array

        :param zoom_level: An integer zoom_level (0 for zoomed all the way out)
        :param tile_position: An n-dimensional array, where n is the number of dimensions
                              in the dataset.
        :param tile_data: The data in the tile.
        c                 S   rK   rL   rC   rE   r   r   r   rA      rN   z-TileSaver.save_tile_array.<locals>.<listcomp>rH   N)minmaxr<   )r-   r"   r:   r;   rI   rJ   r   r   r   save_tile_array   s   	zTileSaver.save_tile_arrayc                 C   s~   t tjtt | dd}t tjtt | dd}t|| jk r3| j|||||d d S | j	|||||d d S )Nr   )Zaxis)rI   rJ   )
rT   npr`   arrayvaluesr_   rS   r(   r^   r[   )r-   r"   r:   r#   rJ   rI   r   r   r   r      s"     

zTileSaver.save_binned_tilec                   C   r/   r'   r   r   r   r   r   r      r2   zTileSaver.flushN)Fr&   )__name__
__module____qualname__r.   r1   r<   r[   r^   ra   r   r   r   r   r   r   r%   3   s    
&r%   c                       s   e Zd Z fddZ  ZS )EmptyTileSaverc                    s   t t| ||| d S r'   )superrh   r.   )r-   r(   r)   r*   	__class__r   r   r.      s   
zEmptyTileSaver.__init__)re   rf   rg   r.   __classcell__r   r   rj   r   rh      s    rh   c                       s,   e Zd Z fddZdd Zdd Z  ZS )ColumnFileTileSaverc                    8   t t| ||||| || _t | _d| _|| _d S Nr   )	ri   rm   r.   	file_pathcsioStringIObulk_txtbulk_txt_lenlog_file)r-   r(   r)   r*   rp   ru   r+   r,   rj   r   r   r.         



zColumnFileTileSaver.__init__c                 C   s   	 |d dkr| j |d d d d d d  n/|d d}| j tt|d d d tt|d d  d tt|d d  d  | j t|d  | j  }|d	krc|   d
S d
S )a]  
        if ('dense' in val['tile_value']):
            value_pos = col.defaultdict(list)
            dense_values = val['tile_value']['dense']
            dense_values = [(x,len(list(y))) for (x,y) in it.groupby(dense_values)]
            dense_values = [item for sublist in dense_values for item in sublist]
            val['tile_value']['dense'] = dense_values
            for i,value in enumerate(dense_values):
                value_pos[value] += [i]
            for key in value_pos:
                sorted_value_pos = sorted(value_pos[key])
                diffs = []
                diffs += [sorted_value_pos[0]]
                for i in range(len(sorted_value_pos)-1):
                    diffs += [sorted_value_pos[i+1] - sorted_value_pos[i]]

                value_pos[key] = diffs
            val['tile_value']['dense'] = value_pos.items()
        r5   Ztileset_info	1r3   r   r   
i N)	rs   writesplitr9   rP   jsondumpstellr   )r-   rY   ticurr_posr   r   r   r1      s,   &	
zColumnFileTileSaver.save_tilec                 C   s   | j  dkrcz t| jd}|| j   W d    n1 s!w   Y  W n; tyb } z/| jd urXt| jd}|tdt	  || W d    n1 sSw   Y  W Y d }~nd }~ww d| _
| j   t | _ d S )Nr   a%Y-%m-%d %H:%M:%S)rs   r~   openrp   rz   getvalue	Exceptionru   r   r   rt   closerq   rr   )r-   Zcolumn_fileexfr   r   r   r     s"   

zColumnFileTileSaver.flush)re   rf   rg   r.   r1   r   rl   r   r   rj   r   rm      s    <rm   c                       sD   e Zd Z							d fdd	Zdd Zdd Zd	d
 Z  ZS )ElasticSearchTileSaverNFc                    rn   ro   )	ri   r   r.   es_pathrq   rr   rs   rt   ru   )r-   r(   r)   r*   r   ru   r+   r,   rj   r   r   r.   $  rv   zElasticSearchTileSaver.__init__c                    s  d|d v r|d d }t t}|D ]&}t| jdkr,|t|d   |d g7  < q||d   |d g7  < qg }| D ]<\}}t|}t| jdkrW|t|g7 }n|t|7 }|tt|g7 }t	t|d D ] | fdd|D 7 }qnq@||d d< | 
|d | d S )Nr\   r6   r   r   c                    s   g | ]}|  qS r   r   )r=   pr>   r   r   rA   V  s    z4ElasticSearchTileSaver.save_tile.<locals>.<listcomp>r5   )coldefaultdictrT   rS   r,   tuplerO   sortedr]   range
save_value)r-   rY   Zsparse_valuesZ	value_posZsparse_valueZvalue_xs_ysr   Zpossr   r   r   r1   :  s&   
 z ElasticSearchTileSaver.save_tilec                 C   sL   | j d| | j t|d  	 | j  }|dkr$|   dS dS )z
        if ('dense' in val['tile_value']):
            print val['tile_id'], len([x for x in val['tile_value']['dense'] if x > 0])
        z{{"index": {{"_id": "{}"}}}}
ry   @KL N)rs   rz   r7   r|   r}   r~   r   )r-   Zdoc_iddocr   r   r   r   r   \  s   
z!ElasticSearchTileSaver.save_valuec                 C   s   | j  dkrcztd| j d | j  | j W n; tyS } z/| jd urIt| jd}|	t
dt  |	| W d    n1 sDw   Y  W Y d }~nd }~ww d| _| j   t | _ d S d S )Nr   http://z/_bulkr   r   )rs   r~   save_to_elasticsearchr   r   r+   r   ru   r   rz   r   r   rt   r   rq   rr   )r-   r   r   r   r   r   r   ~  s&   

zElasticSearchTileSaver.flush)NNNNNFN)re   rf   rg   r.   r1   r   r   rl   r   r   rj   r   r   #  s    ""r   Fc                 C   s   d}t ||}t |d}| D ]*}|tdd|d iid 7 }|t|d 7 }t|dkr:td| || d}qtd	| t|d
krPtd| || d S d S )Nr   Z_bulkrZ   Z_idr5   ry   r   r   z	bulk_txt:r   )opr   r|   r}   rS   r   r   )	partitionZelasticsearch_nodesZelasticsearch_pathr+   rs   Zes_urlZput_urlrY   r   r   r   save_tile_to_elasticsearch  s   
r   c           	      C   s  d}d}t  }|sztj| |dd}|r td||dt||  d}W nW tyz } zK|d9 }td	|d
||tjd t	
| |dkrptdd}|d| d || |  W d   n1 scw   Y  tdtjd  W Y d}~nd}~ww |r
dS dS )a  
    Save some data to elastic search.

    The data should be a string suitable for bulk import by
    elasticsearch. The url should be the location of the index, document
    type, along with the _bulk destination.

    :param url: The elasticsearch url that will ingest the data
                e.g. localhost:9200/hg19/tiles/_bulk
    :param data: The data to import.
                e.g. {"index": {"_id": "blah", "my_json": {"x": 2}}}
    Fr      )datar   z
Savedz
len(data):T   z!
Error saving to elastic search (z), sleeping:fileiX  zunsaved.errr   zUNSAVED url:ry   NzSlept too long, returning)slugidnicerequestspostr   rS   r   sysstderrtimesleepr   rz   r   )	urlr   r+   savedZto_sleepuidrr   r   r   r   r   r     s@   


r   c           
   
   C   s,  | d }| d }t |dtt|}t |}t |s@zt| W n ty? } zt	d|t
jd W Y d}~nd}~ww ddtt||di}|rtt|d	 d
}	|	tj|dd W d   dS 1 smw   Y  dS t|d
}	|	tj|dd W d   dS 1 sw   Y  dS )z
    Save a tile to a particular base directory.

    This function create the appropriate sub-directory based on the
    key.

    They key should be in the format (zoom_level, pos1, pos2...)
    e.g. (5,4,5)
    r   r   r3   zError making directories:r   N_sourcer4   z.gzwr   )indent)r   r   r8   r9   dirnameexistsr   makedirsOSErrorr   r   r   gzipr   rz   r|   r}   )
r0   
output_dirZgzip_outputkeyr6   ZoutpathZoutdirZoeZoutput_jsonr   r   r   r   r1     s(   


""r1   )F)%
__future__r   collectionsr   	cStringIOrq   ImportErrorior   	itertoolsrU   r|   numpyrb   r   Zos.pathpathr   r   r   r   r   r   r   r	   r   r   r   r$   objectr%   rh   rm   r   r   r   r1   r   r   r   r   <module>   s@     cp

0