U
    NZf[{                  	   @   s  d dl Zd dlZd dlZd dlZd dlZd dlmZmZ d dl	Z
d dlZd dlZd dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZmZmZ dZd dlm Z m!Z! ej"j#Z$dd Z%d	d
 Z&dd Z'dd Z(dd Z)dd Z*ej"+ddddhe, fe, ddhfdhddhfe, dhfdfddhddhfe, dhfdfgd d! Z-d"d# Z.d$d% Z/d&d' Z0ej"+d(d)d*gd+d, Z1d-d. Z2d/d0 Z3d1d2 Z4d3d4 Z5d5d6 Z6d7d8 Z7d9d: Z8d;d< Z9ej"+d=d*d)gd>d? Z:d@dA Z;dBdC Z<dDdE Z=dFdG Z>dHdI Z?dJdK Z@dLdM ZAdNdO ZBej"+dPdQdRdSgdTdU ZCdVdW ZDdXdY ZEdZd[ ZFd\d] ZGd^d_ ZHd`da ZIdbdc ZJddde ZKdfdg ZLdhdi ZMdS )j    N)catch_warningssimplefilter)		DataFrameDatetimeIndexIndex
MultiIndexSeries	Timestampconcat
date_rangetimedelta_range)_maybe_removeensure_clean_store
safe_closeZbloscHDFStoreread_hdfc              
   C   s   t | <}z t|}tdW 5 Q R X W n tk
r@   Y nX W 5 Q R X t | J}t|6}t  |d< t|dks~tt|d tkstW 5 Q R X W 5 Q R X d S )NZblaha   )	tmensure_cleanr   
ValueErrormakeDataFramelenAssertionErrortyper   )
setup_pathpathZtbl r   G/tmp/pip-unpacked-wheel-nbcvw55c/pandas/tests/io/pytables/test_store.pytest_context+   s    

r    c                    st   t jdfdd  fdd}|| dd}|| dd}td	 || dd}|| dd}||ksdt||ksptd S )
N   c              	      sJ   | t | d,t fdddD ]}| q(W 5 Q R X  S )Nrbc                      s     j S N)read
block_sizer   chunk_num_blocksfhr   r   <lambda>@       z7test_no_track_times.<locals>.checksum.<locals>.<lambda>r+   )openiterupdatedigest)filenameZhash_factoryr'   chunkr   r&   r   checksum=   s
    z%test_no_track_times.<locals>.checksumc              	      sL   |  }t ddgi}t|dd}|jd|ddd |d W 5 Q R X  |S )Nr   r   wmodetableT)formatdata_columnsindextrack_times)r   r   put)tmp_pathr:   r   dfZhdfr2   r   r   r   create_h5_and_return_checksumD   s    	z:test_no_track_times.<locals>.create_h5_and_return_checksumF)r:   Tr   )hashlibmd5timesleepr   )r<   r   r?   Zchecksum_0_tt_falseZchecksum_0_tt_trueZchecksum_1_tt_falseZchecksum_1_tt_truer   r>   r   test_no_track_times9   s    
rD   c              	   C   s(   t | }t|g kstW 5 Q R X d S r#   )r   listr   r   storer   r   r   test_iter_emptyd   s    
rH   c              
   C   s  t | `}t| |  t |d< t |d< t |d< t }d|d< d|d< |d d	k|d
< |d d	k|d< d|d< d|d< d|d< td|d< td|d< t	dddd	d	|d< t	dddd	d	|d< t
j|j|jdd dgf< | }tdd tdtjj ||d< W 5 Q R X |j|jjd |jt|ksJt|jt|ks^t|  W 5 Q R X t | 4}t }|d| |d}t| t| W 5 Q R X d S ) Nr   bcfooobj1barobj2Ar   bool1Bbool2Tbool3r   int1   int220010102
timestamp120010103
timestamp2  	datetime1   	datetime2   recordignorer=   Zbah)r   reprinfor   makeTimeSeriesZmakeStringSeriesr   r	   dtdatetimenpnanlocr9   _consolidater   r   pderrorsZPerformanceWarning_handleZcreate_grouprootr0   r   strappend
get_storer)r   rG   r=   sr   r   r   	test_reprj   sB    

rt   c              
   C   s   t | }t |d< t |d< t |d< d|ks:td|ksFtd|ksRtd|ks^td|ksjtd|ksvtd|ksttdd	 t |d
< W 5 Q R X d
|kstW 5 Q R X d S )Nr   rI   zfoo/barrJ   z/foo/barz/foo/brM   Tr`   znode()))r   r   re   r   r   r   rF   r   r   r   test_contains   s    
ru   c              
   C   s   t | }t |d< t |d< t }t|d |d|d d  |d|dd   |jjj	j
dkspt|jjj	j
dkst|jjj	j
dkstt|d |d| d |dj	_
d}tjt|d |d W 5 Q R X W 5 Q R X d S )	Nr   rI   df1
   z0.15.2df2z/'NoneType' object has no attribute 'startswith'match)r   r   re   r   makeTimeDataFramer   rq   ro   r   Z_v_attrsZpandas_versionr   rI   rv   Zget_nodepytestraises	Exceptionselect)r   rG   r=   msgr   r   r   test_versioning   s     


r   zwhere, expected/Zfirst_groupZsecond_grouprv   rx   Zthird_groupdf3s1df4) /first_group/second_group/second_group/third_groupr   )r   r   c                 C   s  t dddgt dddgt dddgt d	d
dgtd
d	dgtdddgdddggtjddgddtjddgddd}tddd`}|d|d  |d|d  |d|d  |d|d  |d|d  |jd d!|d!  |jjd d"|d" d# |jjd$d%|d% d# t	t
|j| d&t	|ks<t|j| d&D ]\}}}||ks`t|| \}}|t|ks~t|t|kst|D ]J}	d'||	g}
||
}d(|	krt|||	  nt|||	  qqHW 5 Q R X d S ))Nr   rU   r]         r_         	   rw      )r   rU   r]   )r   r   r_   zi,i,iZdtype)r   r   r   )rw   r      )rv   rx   r   r   r   a1tb1tb2zwalk_groups.hdfr3   r4   z/first_group/df1rv   z/first_group/df2rx   z/second_group/df3r   z/second_group/s1r   z/second_group/third_group/df4r   r   r   r   )objr   r   wherer   r=   )r   r   rh   arrayr   r;   rn   Zcreate_arrayZcreate_tabler   rE   walkr   setjoingetr   assert_frame_equalassert_series_equal)r   expectedobjsrG   r   groupsZleavesZexpected_groupsZexpected_framesZleafZ
frame_pathr   r   r   r   	test_walk   s<    "

r   c                 C   s   t | }t }||d< |j}t|| t|d}t|| t }||d< |j}t|| dD ]4}d| d}t	j
t|d t|| W 5 Q R X qhdD ]}t|d|  qW 5 Q R X d S )	Nr   r=   )dr5   r   handlecomplibz$'HDFStore' object has no attribute ''ry   )r5   r   r   r   _)r   r   re   r   r   getattrr{   r=   r   r|   r}   AttributeError)r   rG   rs   resultr=   xr   r   r   r   test_getattr   s"    

r   c                 C   s   t dtjdgdtjtjgdtdd}t ddgdtjgdtdd}| | }|j|dd	d
 t|d}t|| | | }|j|dd	dd t|d}t|| | | }|j|dd	dd t|d}t|| d S )Ng        g       @g      ?)Zcol1Zcol2abcr9   acr=   r6   r7   F)r7   ZdropnaT)r   rh   ri   rE   to_hdfr   r   r   )r<   r   Zdf_with_missingZdf_without_missingr   Zreloadedr   r   r   test_store_dropna  s(     


r   c                 C   s   | | }t  d}|j|ddddid |  jddd}|j|dd	dd
 t t|dt	||g |d j|ddddid |d j|dd	dd
 t 
t|dt	|d |d g d S )NCZss3r6   r9   r_   )r7   Zmin_itemsizeZlonger)r   T)rq   r7   rQ   Zss4)r   ZmakeMixedDataFrame	set_indexr   copyZreset_indexZassignr   r   r
   r   )r<   r   r   r=   rx   r   r   r   test_to_hdf_with_min_itemsize6  s    r   r7   fixedr6   c                 C   sN   dg}t |t|d}| | }|j|d|dd t|ddd}t|| d S )Nu   foor   r6   surrogatepass)r7   rm   )rm   )r   r   r   r   r   r   )r<   r7   r   dataZserr   r   r   r   r   test_to_hdf_errorsG  s    r   c                    s8  t | $ tdd  fdd}t }d|d< d|d<  jd	|ddgd
 |d	djdksft|d	djdkszt|d	djdkst jd|dgddgd |ddjdkst|ddjdkst|ddjdkstt d  d| d}t	j
t|d  d W 5 Q R X W 5 Q R X W 5 Q R X d S )NTr`   c                    s   t  | jj|S r#   r   rr   r6   colstcolumnrG   r   r   colW  s    z$test_create_table_index.<locals>.colrK   stringrM   string2r(   r8   r9   f2)r9   r8   Fz1cannot create table index on a Fixed format storery   )r   r   r   r{   rq   
is_indexedr   r   r;   r|   r}   	TypeErrorcreate_table_indexr   r   r=   r   r   r   r   test_create_table_indexS  s&    
r   c                    s   t | ҉ tdd  fdd}t }d|d< d|d<  jd	|dgd
 |d	djdks`t|d	djdksttd}tjt	|d |d	dj W 5 Q R X d}tjt	|d  j
d	dgd W 5 Q R X W 5 Q R X W 5 Q R X d S )NTr`   c                    s   t  | jj|S r#   r   r   r   r   r   r   w  s    z:test_create_table_index_data_columns_argument.<locals>.colrK   r   rM   r   r(   r   r9   z('Cols' object has no attribute 'string2'ry   zcolumn string2 is not a data_column.
In order to read column string2 you must reload the dataframe 
into HDFStore and include string2 with the data_columns argument.columns)r   r   r   r{   rq   r   r   r|   r}   r   r   r   r   r   r   -test_create_table_index_data_columns_argumentq  s     
r   c              	   C   s   t jtdddtdgddgd}tddd	d
ddgi|d}t| D}|jd|dd |jddd}|jdgd d f }t	
|| W 5 Q R X d S )Nz
2000-01-01r   periodsdateid)namesr   g?g333333?g?gffffff?g      ?r   r=   Tr   zid == 1r   r   )r   Zfrom_arraysr   ranger   r   rq   r   ilocr   r   )r   idxr=   rG   actualr   r   r   r   test_mi_data_columns  s     
r   c              	   C   s   t  }d|d< d|d< |d dk|d< |d dk|d	< d
|d< d|d< d|d< td|d< td|d< tddddd|d< tddddd|d< tj|j|jdd dgf< |	 }t
| $}|d| t |d| W 5 Q R X d S )NrK   rL   rM   rN   rO   r   rP   rQ   rR   TrS   r   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   Z	df1_mixed)r   r   r	   rf   rg   rh   ri   rj   r9   rk   r   rq   r   r   r   r=   rG   r   r   r   test_table_mixed_dtypes  s"    
r   c           	   	   C   s   d}dt dddtdg}tjj||d}t ddd	}t|d|d
}t|j	|
td }t| J}|d| |d}t|| |d| |d}t|| W 5 Q R X d S )NzSun Mon Tue Wed Thuz
2012-05-01i  r   r   z
2014-05-01)holidaysZweekmaskr      )r   freqzMon Tue Wed Thu Fri Sat Sunr   r6   )rf   rg   rh   Z
datetime64rl   offsetsZCustomBusinessDayr   r   weekdaymapsplitr   r;   r   r   r   rq   )	r   Zweekmask_egyptr   Z
bday_egyptZmydtZdtsrs   rG   r   r   r   r   test_calendar_roundtrip_issue  s&     


r   c              
   C   s.  t | }t }t }||d< ||d< t|d t|dksFtt||d  t|d t|dkspttj	t
dd |d W 5 Q R X ||d< ||d< t|d	 t|d t|dkst||d< ||d< t|d t|dkst||d< ||d< |d= |d= t|dks tW 5 Q R X d S )
Nr   rI   r   r   z1'No object named a_nonexistent_store in the file'ry   Za_nonexistent_storezb/foorK   )r   r   re   r   r   r   r   r   r|   r}   KeyErrorremove)r   rG   tsr=   r   r   r   test_remove  s:    

 


r   c              	   C   s   t | }ttjddtdddd}|jd|dd ||jtd	k }|	dd
}t
|| |	dd
}t
|| |	dd}t
|| W 5 Q R X d S )N   rU   20130101r   r   r=   r6   r   Z20130105z!index>datetime.datetime(2013,1,5)zindex>datetime(2013,1,5))r   r   rh   randomrandnr   r;   r9   r	   r   r   r   )r   rG   r=   r   r   r   r   r   test_same_name_scoping  s    
r   c              	   C   sD   t  }d|j_t| "}||d< |d }t || W 5 Q R X d S )NrK   frame)r   r   r9   namer   r   )r   r=   rG   reconsr   r   r   test_store_index_name  s    
r   table_formatc              	   C   s   t ttdddtdddgdd}t ttdddtdddgdd}ttddd||d	}| | }|j	|d
|d t
|d
}tj||dd t|jjtkstt|jjtkstd S )Ni  r   rU   u   colsג)r   i  u   rowsאr   r   r9   r=   r   T)Zcheck_names)r   rl   Zto_datetimerf   r   r   rh   arangeZreshaper   r   r   r   r   r9   r   rp   r   r   )r<   r   r   r   Zidx1r=   r   rx   r   r   r   test_store_index_name_numpy_str  s      
r   c              	   C   sD   t  }|d }t| "}||d< |d }t || W 5 Q R X d S )NrO   series)r   r   r   r   )r   r=   r   rG   r   r   r   r   test_store_series_name-  s    
r   c              	   C   sD   t | 2}t |d< t }||d< t|d | W 5 Q R X d S )Nr   )r   r   r{   re   r   )r   rG   r   r   r   r   test_overwrite_node7  s
    
r   c              
   C   sh  t  }t| }t|d |d| |d}|jtt	|j
k sRtt|d ttdtdd}|d| |ddg}|jtdk st|jd|d}|jddd d f }t || |dd	d
g}|jtdd k st|jd|d}|jddd d f }t || t|tsBtt|d t|d t  }t  jdjd}|jd|ddgd |d| |dddg}|d|}|d|}	t||	gdd}t||gdd}||jdk|jdk@  }t j||dd W 5 Q R X t| J}ttjddtdddd}|d| |dd}|t|jdk j
}
|j|
 }|jd|
d}t || |jd|
d}t || d}t j!t"|d" |jdtjt	|d d!d W 5 Q R X t j!t"|d" |jdtt	|d d W 5 Q R X t j!t"|d  |jdtt	|dd" W 5 Q R X t j!t"|d" |jdtt	|dd#d$ W 5 Q R X tdd%d}|jdd&d}||j
#| }t || ttjd#d}|d| |jddddgd}|jdddg }t || d'gd# }
d|
d(< |jd|
d}|j|
 }t || |jddd#d)}|dd# }t || W 5 Q R X d S )*Nr=   r   rO   rQ   zindex<3r]   r   r   rU   zindex>=3zindex<=4r   rv   rx   z{}_2r   rO   rQ   r   zA>0zB>0r   )ZaxisF)Z
check_freqi  Z20000101r   r   r9   zLwhere must be passed as a string, PyTablesExpr, or list-like of PyTablesExprry   Zfloat64r   )r   startrw   )r   r   stopi  zindex in selectionTr   r   )$r   r{   r   r   rq   Zselect_as_coordinatesvaluesrh   r   r   r9   allr   r   r   r   rj   r   
isinstancer   renamer7   r
   rO   rQ   r   r   r   Zselect_columnr   monthr   r|   r}   r   isin)r   r=   rG   rJ   r   r   rv   rx   Z
df1_resultZ
df2_resultr   r   Z	selectionr   r   r   test_coordinates@  s    




 

&&$&

r  c              	   C   s   t | }ttjdtjdd}|d| |jddddd}|jddd	gf }t	|| |jddd
dd}t
|dkst|jd
dd	gf }t	|| W 5 Q R X d S )Nr   r   r=   zcolumns=['A']r   r   r   r   rO   r   (   )r   r   rh   r   randrq   r   rj   r   r   r   r   r   rG   r=   r   r   r   r   r   test_start_stop_table  s    
r  c              	   C   s|   t | j}tddgddgd}|jdgd d|dd |jddgdd	dd
}|jd	gddgf }t|| W 5 Q R X d S )Nr   rU   )rK   rM   rK   )selectorr   r  )r  r   r   )r  r   r   rM   )r   r   Zappend_to_multipleZselect_as_multiplerj   r   r   r  r   r   r   test_start_stop_multiple  s    

     r  c              	   C   s  t | l}ttjdtjddtdddd}|d| |jdddd	}|jddd d f }t	
|| |jddd
d	}|jdd
d d f }t	
|| |jdddd	}|jddd d f }t	
|| |j}|d| |jdddd	}|jdd }t	|| |jddd
d	}|jdd
 }t	|| t	 }tj|jddddf< tj|jdd
df< W 5 Q R X d S )Nr   r   r   r   r   r=   r   r   r   rw   r   r  rs   r]   r   r   r   )r   r   rh   r   r  r   r;   r   r   r   r   rO   r   r   ri   )r   rG   r=   r   r   rs   r   r   r   test_start_stop_fixed  s4    
r  c              
   C   s   t tjdd}dd |jD |_dd |jD |_t| }|jd|dd d	}|d|g}t	
||jd d |jd d
 f  d}|d|g}t	
||jd d |jd d
d f  W 5 Q R X d S )N2   d   c                 S   s   g | ]}|d qS Z3dr   .0rJ   r   r   r   
<listcomp>  s     z-test_select_filter_corner.<locals>.<listcomp>c                 S   s   g | ]}|d qS r  r   r  r   r   r   r    s     r   r6   r   zcolumns=df.columns[:75]K   zcolumns=df.columns[:75:2]rU   )r   rh   r   r   r9   r   r   r;   r   r   r   rj   )r   r=   rG   critr   r   r   r   test_select_filter_corner  s    
$r  c                     s0   t   t  fdddd } t  |  d S )Nc                    s     | dS Nr=   r   pr=   r   r   r*   	  r+   z#test_path_pathlib.<locals>.<lambda>c                 S   s
   t | dS r  r   r  r   r   r   r*   	  r+   r   r   Zround_trip_pathlibr   r   r   r  r   test_path_pathlib  s    
 r  zstart, stop)r   rU   )r   rU   )NNc              	   C   sj   t tdddgtdddgd}t|6}|d| |jd| |d}t|| | | W 5 Q R X d S )	Ni2i2i2abZcdr   rI   Ztest_datasetr   )r   r   r   rq   r   r   r   )r   r   r   r=   rG   r   r   r   r    test_contiguous_mixed_data_table  s    
r  c                     s8   t    fdd} dd }t | |}t  | d S )Nc              	      s$   t | } |d W 5 Q R X d S r  r   r   r   rG   r  r   r   writer"  s    
z*test_path_pathlib_hdfstore.<locals>.writerc              
   S   s*   t | }t|dW  5 Q R  S Q R X d S r  r   r  r   r   r   reader&  s    
z*test_path_pathlib_hdfstore.<locals>.readerr  r   r!  r   r   r  r   test_path_pathlib_hdfstore  s
    r#  c                     s0   t   t  fdddd } t  |  d S )Nc                    s     | dS r  r  r  r  r   r   r*   1  r+   z,test_pickle_path_localpath.<locals>.<lambda>c                 S   s
   t | dS r  r  r  r   r   r   r*   1  r+   r  r  r   r  r   test_pickle_path_localpath.  s    
 r$  c                     s8   t    fdd} dd }t | |}t  | d S )Nc              	      s$   t | } |d W 5 Q R X d S r  r  r  r  r   r   r   9  s    
z,test_path_localpath_hdfstore.<locals>.writerc              
   S   s*   t | }t|dW  5 Q R  S Q R X d S r  r   r  r   r   r   r!  =  s    
z,test_path_localpath_hdfstore.<locals>.reader)r   r   Zround_trip_localpathr   r"  r   r  r   test_path_localpath_hdfstore6  s
    r%  c                  C   s|   t ddh ddd} t }t B}t|}|jd|dgd W 5 Q R X | |d | |d	d
 W 5 Q R X W 5 Q R X d S )NTr`   c                 [   s  |d krt  \}}zt| d}|j	|f||d|}|d krH|
 }t|t|
 ks`t|
 D ]Z}||jrh||}	||}
|
j|	jkst|rh|
jD ]}|jr|	|j jstqqhW 5 t| t| zt| W n ttfk
r   Y nX t| X d S )Nr)keyspropindexes)tempfilemkstempr   oscloseOSErrorr   r   r   r   r'  r   r   rr   Zis_tableZnrowsZaxesr   r   )r(   new_fr'  r(  kwargsfdrG   ZtstorekZnew_tZorig_tr   r   r   r   do_copyH  s0    



ztest_copy.<locals>.do_copyr=   rO   r   )r(   F)r(   r(  )NNT)r   r   r   r   r   rq   )r2  r=   r   str   r   r   	test_copyE  s    
%


r4  c              	   C   s   t ddgddggd}| | }d}tjt|d |j|ddd W 5 Q R X |j|dd	d t|d}t|| ||st	||st	d S )
Nr   r   )r   r   z/Columns index has to be unique for fixed formatry   r=   r   r   r6   )
r   r|   r}   r   r   r   r   r   equalsr   )r<   r   r=   r   r   otherr   r   r   test_duplicate_column_namev  s    
r7  c              	   C   sT   t tjjdd}tddddd|_t| }||d< t|d | W 5 Q R X d S )	N)rw   r   )sizeZ0srw   Z1sZexample)r   r   r   r   r=   )	r   rh   r   normalr   r9   r   r   r   r   r   r   r   !test_preserve_timedeltaindex_type  s
    
r:  c                 C   s   t tjddtdtdd}d|j_|jddd	}|jj|j	
  }| | }|j|d
dd|dd td}t|}t|d
|d ||kstd S )Nr   r   ZabcdZABCDE)r9   r   lettersET)r'  rq   r=   r   F)r5   rq   r8   r9   ZBCDr   )r   rh   r   r  rE   r9   r   r   r   r   tolistr   r   r   )r<   r   r=   r8   r   Z	cols2loadZcols2load_originalr   r   r    test_columns_multiindex_modified  s"     r>  c           	      C   s  t jt jt jt jt jg}t jt jg}|D ]l}tt	j
dd|dd}| | }tdd6 d}tjt|d |j|dd	dd
 W 5 Q R X W 5 Q R X q(|D ]x}tt	j
dd|dd}| | }tddB |j|dd	dd
 t|dd|jd  dd}t|stW 5 Q R X qd S )Nrw   rU   r   Tr`   z-cannot have non-object label DataIndexableColry   r=   r6   )r7   r8   z	index = [r   ]r   )r   ZmakeIntIndexZmakeFloatIndexZmakeDateIndexZmakeTimedeltaIndexZmakePeriodIndexZmakeStringIndexZmakeCategoricalIndexr   rh   r   r   r   r|   r}   r   r   r   r9   r   r   )	r<   r   Ztypes_should_failZtypes_should_runr9   r=   r   r   r   r   r   r   $test_to_hdf_with_object_column_names  s.    (r@  c              	   C   sb   t ddddgddddgd	}t| 2}|d
| |d jj|d
 d jjksTtW 5 Q R X d S )Nr   rU   r]   r   r   r_   r   r   r  r=   r   )r   r   r;   r   stridesr   r   r   r   r   test_hdfstore_strides  s    
rB  c                 C   sT   t dggdgtdgddd}| }| | }||d t|d}t|| d S )Nr   TFboolr   r   r   )r   r   r   r   r   r   r   )r<   r   r=   r   r   r   r   r   r   test_store_bool_index  s    
rD  )Nrg   rf   r@   r+  r)  rB   warningsr   r   Znumpyrh   r|   Zpandasrl   r   r   r   r   r   r	   r
   r   r   Zpandas._testingZ_testingr   Zpandas.tests.io.pytables.commonr   r   r   Z_default_compressorZpandas.io.pytablesr   r   markZ
single_cpuZ
pytestmarkr    rD   rH   rt   ru   r   Zparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r#  r$  r%  r4  r7  r:  r>  r@  rB  rD  r   r   r   r   <module>   s   ,+,



(
'


	m'	
1
 