U
    NZfj                  
   @   s(  d dl mZmZ d dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
 d dlZd dlZd dlmZmZ d dlm  mZ d dlZd dlmZmZmZmZmZmZmZ d dlmZ  d d	l!m"Z" d d
l#m$Z$ G dd dZ%ej&'dej(d dd Z)ej&'deegej&'dej(d dd Z*dd Z+dd Z,dd Z-ej&'ddddgdddggdd Z.d d! Z/ej&'d"e 0d#e1d d$ie j2e1d d$d#d%gd&d' Z3d(d) Z4d*d+ Z5ej&'d,d-d.d/d0gd1d2 Z6ej&'d3d4gd5 d6fd4gd5 d7fd4d8d4gd7fd4d8d4gd6fgd9d: Z7d;d< Z8d=d> Z9ej&'d?ed@dAgdBedCd@dAgidBee:dDee:dDgdEdF Z;ej&'d?ed@dAgdBedCd@dAgidBee:dDee:dDgdGdH Z<ej=ej&'dIdJdKdLgej&'dMddJdNgdOdP Z>ej=ej&'dIdJdKdLgej&'dMddJdNgdQdR Z?ej=dSdT Z@dS )U    )abcdeque)datetime)Decimal)Iterator)catch_warningssimplefilterN)InvalidIndexErrorPerformanceWarning)	DataFrameIndex
MultiIndexPeriodIndexSeriesconcat
date_range)SparseArray)
to_decimalc                   @   s   e Zd Zdd Zdd Zdd Zdd Zej	d	d	d
gdd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1S )2TestConcatenatec                 C   s   t dddd}t dddd}ttjd|}ttjd|}| }| }t||g}t|jt	slt
|jd |jd kst
d S )	Nz
12/31/1990z
12/31/1999zA-DEC)freqz
12/31/2000z
12/31/2009
   r   )r   r   nprandomrandnZ	to_periodr   
isinstanceindexr   AssertionError)selfd1Zd2s1s2result r"   K/tmp/pip-unpacked-wheel-nbcvw55c/pandas/tests/reshape/concat/test_concat.pytest_append_concat&   s    z"TestConcatenate.test_append_concatc                    s&  t tjdd}t tjjdddddd}t dditdd	}t|||gdd
d}|sz|jj	D ]  j
d ksdtqdn|jj	D ]  j
d k	stqt|||gddd}|jj	D ]  jjdkr܈ j
|jj	d j
kstq jjdkr j
|jj	d j
ks<tq jtkr|r. |jj	d ks<tq j
d k	stqt tjdd}t||||gddd}|jj	D ]  jjdkr|s|rt fdd|jj	|jj	 D stn j
d kstnV jjdkr j
|jj	d j
kstn, jtkrnt fdd|jj	D sntqnd S )N      r   r   )size      foor   T)axiscopyFf)iuc                 3   s   | ]}t  |V  qd S Nr   Zshares_memory.0otherarrr"   r#   	<genexpr>[   s   z3TestConcatenate.test_concat_copy.<locals>.<genexpr>c                 3   s   | ]}t  |V  qd S r1   r2   r3   r6   r"   r#   r8   f   s     )r   r   r   r   randintreshaperanger   Z_mgrarraysbaser   dtypekindobjectany)r   Zusing_array_managerZusing_copy_on_writedfdf2Zdf3r!   Zdf4r"   r6   r#   test_concat_copy6   s@    
z TestConcatenate.test_concat_copyc              
   C   s  t tjdd}t tjdd}t||gddgd}tdddddddgdddddddgg}t tj|j|jf |d}t	
|| t||gddgd}tddddddgddddddgg}t tj|j|jf |d}t	
|| t tjdd}t tjdd}t||gddgdd}t tj|j|jf |d	}t	
|| t||gddgdd}t tj|j|jf |d	}t	
|| d S )
Nr&   r%   r   r(   keys   r+   rF   r,   columns)r   r   r   r   r   r   Zfrom_arraysZr_valuestmassert_frame_equalZc_)r   rB   rC   r!   	exp_indexexpectedZ
exp_index2r"   r"   r#   test_concat_with_group_keysh   s(    "&z+TestConcatenate.test_concat_with_group_keysc                 C   s   t tjdd}|jd d ddgf |jd d dgf |jd d dgf g}ddd	d
g}t|dd	ddg|gdgd}t|jj	d t
|dd t|jj	d t
ddddg |jjdd gkstd S )Nr   r%   r   r(   rG   r&   ZthreetwooneZzeroZ	group_key)r,   rF   levelsnamesname)r   r   r   r   ilocr   rL   assert_index_equalrJ   rS   r   rT   r   )r   rB   pieceslevelr!   r"   r"   r#    test_concat_keys_specific_levels   s    < z0TestConcatenate.test_concat_keys_specific_levelsmappingdictc              	      s  |dkrt n|}|ttjddttjddttjddttjddd t  }t }t fdd|D |d}t	|| t dd	}t fd
d|D |dd}t	|| dddg}t |d}t fdd|D |d}t	|| d S )Nr]   r%   r&   )r*   barbazZquxc                    s   g | ]} | qS r"   r"   r4   kframesr"   r#   
<listcomp>   s     z7TestConcatenate.test_concat_mapping.<locals>.<listcomp>rE   r(   r,   c                    s   g | ]} | qS r"   r"   r`   rb   r"   r#   rd      s     rH   r_   r*   r^   c                    s   g | ]} | qS r"   r"   r`   rb   r"   r#   rd      s     )
r]   r   r   r   r   listrF   r   rL   rM   )r   r\   Znon_dict_mapping_subclassconstructorZsorted_keysr!   rO   rF   r"   rb   r#   test_concat_mapping   s&    	
z#TestConcatenate.test_concat_mappingc                 C   s>  t tjdd}t tjdd}ddgddgg}dd	g}t||||gd
dddg||d}t||||g}t|dgg ddddgddddgddddgg|d g d}||_t|| t||||gd
dddg|d}|jj	dkst
t||||gd
dddgdd	gd}|jj	dkst
t|jjd tddgdd d S )Nr(   r&   r%   r*   r_   rR   rQ   firstsecond)r*   rR   )r*   rQ   )r_   rR   )r_   rQ   )rF   rS   rT   r   )rS   codesrT   rF   rS   )NNNrF   rT   )ri   rj   NrU   )r   r   r   r   r   r   r   rL   rM   rT   r   rX   rS   r   )r   rB   rC   rS   rT   r!   rO   rN   r"   r"   r#   test_concat_keys_and_levels   sD    


 




 z+TestConcatenate.test_concat_keys_and_levelsc              	   C   s   t tjdddgd}t tjdddgd}d}tjt|d$ t||gd	d
gdddggd W 5 Q R X d}tjt|d$ t||gd	d
gdddggd W 5 Q R X d S )Nr(   r&   ar+   r%   bz Values not found in passed levelmatchrR   rQ   r*   r^   r_   rl   zKey one not in level)r   r   r   r   pytestraises
ValueErrorr   )r   rB   rC   msgr"   r"   r#   "test_concat_keys_levels_no_overlap   s    (z2TestConcatenate.test_concat_keys_levels_no_overlapc              	   C   sb  ddddg}t tjddddgd	d
tjddddgdd
tjddddgd	d
tjddddgdd
d|d}t tjddddgdd
tjddddgd	d
tjddddgdd
tjddddgd	d
d|d}t||gdd}t tj|j|jgdd|d}t|| t tj	dddgd}t tj	dddgd}t||gddgddgd}|j
jdks^td S )NABCDr(   rG   r&   r%   Zf8r>   i8)rx   ry   rz   r{   rI   Tignore_indexr   re   ro   r+   rp   rR   rQ   ri   rj   rm   )ri   rj   )r   r   arrayr   ZconcatenaterK   rL   rM   r   r   r   rT   r   )r   rJ   df1rC   ZappendedrO   rB   r!   r"   r"   r#    test_crossed_dtypes_weird_corner   s6    

 z0TestConcatenate.test_crossed_dtypes_weird_cornerc                 C   s@   t dddtdd}t dddtdd}t||g|d d S )Nr*   r^   )rx   ry   r(   rG   r+   )ry   r   sort)r   r;   r   )r   r   r   rC   r"   r"   r#   test_with_mixed_tuples  s    z&TestConcatenate.test_with_mixed_tuplesc                 C   s  t dddd}tjddd}t||d}t||d}t|dd	|d}tt|d
dd
|ddgd}t||gd	d}t	|| tt|d
dd
|dd	gd}t||gd	d}t	|| tt|ddd|dd	d
gd}t|||gd	d}t	|| tt|ddd|ddd	d
dgd}t|||||gd	d}t	|| d|_
tt|ddd|dddgd}t|||gd	d}t	|| d|_
tt|ddd|dddgd}t|||gd	d}t	|| tt|ddd|dd	d
gd}t|||gd	dd}t	|| tt|ddd	| d dgd}t|||g}t	|| tt|ddd	dgd}t|||gdd}t	|| d S )Nz01-Jan-2013r   H)Zperiodsr   int64r|   r+   r(   rG   r   r   rJ   re   r&   r)   r*   r^   T)r,   r   rI   r~   )r   r   aranger   r   r:   repeatr   rL   rM   rV   Ztiletolist)r   r   r7   r   r    rB   rO   r!   r"   r"   r#   test_concat_mixed_objs  s                   
 z&TestConcatenate.test_concat_mixed_objsc                 C   s   t dtddtjgi}t|jdg |jdg g}t|j	|j	 t dt
dddt
dddgi}t|jdg |jdg g}t|j	|j	 t d	d
gd gd  i}t|jdg |jdg g}t|j	|j	 d S )NdateZ20130101UTCr   r(   i  i  rG   textz
some words	   )r   pd	TimestampZtz_localizeNaTr   rW   rL   assert_series_equalZdtypesr   )r   rB   r!   r"   r"   r#   test_dtype_coerceionU  s     z$TestConcatenate.test_dtype_coerceionc                 C   sN   t tjdd}t|gdgd}t||gddgd}t||d d  d S )Nr   r%   r*   rE   r^   )r   r   r   r   r   rL   rM   )r   rB   r!   rO   r"   r"   r#   test_concat_single_with_keyf  s    z+TestConcatenate.test_concat_single_with_keyc              	   C   s&   t jtdd tg  W 5 Q R X d S )NzNo objects to concatenaterq   )rs   rt   ru   r   r   r"   r"   r#   test_concat_no_items_raisesm  s    z+TestConcatenate.test_concat_no_items_raisesc              	   C   sl   t tjdd}|d d d d |dd  g}t|}t|| tjt	dd td d g W 5 Q R X d S )Nr   r%   r)   zAll objects passed were Nonerq   )
r   r   r   r   r   rL   rM   rs   rt   ru   )r   rB   rY   r!   r"   r"   r#   test_concat_exclude_noneq  s    z(TestConcatenate.test_concat_exclude_nonec                 C   s   t dddgdddgdddgg}td ||d d |d d |d}t||d d |d d |d}t|| td ||d d |d d |gdd	d
ddgd}t||d d |d d |gd	d
ddgd}t|| d S )Nr         rG   r(   )ro   rp   cde)rp   r   r   r   ro   rp   r   r   r   rE   r   r   rL   rM   )r   Zdf0r!   rO   r"   r"   r#   test_concat_keys_with_nonez  s     $" ,z*TestConcatenate.test_concat_keys_with_nonec                 C   sV   t  }t  d d d }t||gddd}t||gddd}t|t|ksRtd S )NrG   outerr(   )joinr,   )rL   ZmakeTimeSeriesr   lenr   )r   ts1Zts2leftrightr"   r"   r#   test_concat_bug_1719  s
    z$TestConcatenate.test_concat_bug_1719c                 C   s`   t td}t td}d |_|_t||gdd}t||d}ddg|_t	|| d S )Nr)   z	same namer(   re   r   r(   )
r   r   zerosZonesrV   r   r   rJ   rL   rM   )r   Zts0r   r!   rO   r"   r"   r#   test_concat_bug_2972  s    
z$TestConcatenate.test_concat_bug_2972c              
   C   s   t ddddgddddgddddgd}t ddddgd	d
ddgddddgd}t ddddd	dgddddd
dgddddddgddddddgg}ddddddg|_t||gd	d}t|| d S )Nr      Zrrr)firmNoprc	stringvarr   r         r(   rG   r&   r%   )rz   miscr   r   r   r   rz   r   re   )r   rJ   r   rL   rM   )r   r   rC   rO   r!   r"   r"   r#   test_concat_bug_3602  s&    


"z$TestConcatenate.test_concat_bug_3602c                    s   t dddg t dddgt ddddddg}tt fdd| tt gdd| ttd	d
  fD dd| ttt fdd| G  fddd}tt| dd| G  fdddtj}tt| dd| d S )Nr(   rG   r&   r%   r)   r   Tr~   c                 s   s   | ]
}|V  qd S r1   r"   )r4   rB   r"   r"   r#   r8     s     z8TestConcatenate.test_concat_iterables.<locals>.<genexpr>c                       s(   e Zd ZedddZ fddZdS )z>TestConcatenate.test_concat_iterables.<locals>.CustomIterator1returnc                 S   s   dS )NrG   r"   r   r"   r"   r#   __len__  s    zFTestConcatenate.test_concat_iterables.<locals>.CustomIterator1.__len__c              
      s>   z d| W S  t k
r8 } z
t|W 5 d }~X Y nX d S )Nr   )KeyError
IndexError)r   r   errr   rC   r"   r#   __getitem__  s    zJTestConcatenate.test_concat_iterables.<locals>.CustomIterator1.__getitem__N)__name__
__module____qualname__intr   r   r"   r   r"   r#   CustomIterator1  s   r   c                       s    e Zd Zed fddZdS )z>TestConcatenate.test_concat_iterables.<locals>.CustomIterator2r   c                 3   s    V  V  d S r1   r"   r   r   r"   r#   __iter__  s    zGTestConcatenate.test_concat_iterables.<locals>.CustomIterator2.__iter__N)r   r   r   r   r   r"   r   r"   r#   CustomIterator2  s   r   )r   rL   rM   r   r   r   Iterable)r   rO   r   r   r"   r   r#   test_concat_iterables  s     
z%TestConcatenate.test_concat_iterablesc                 C   sZ   t tdddd gdg}|dd tdD 7 }t|dd	j}tddd g}t|| d S )
Nr&   ro   r(   r   c                 S   s"   g | ]}t td dddgdqS )r&   Nr(   ro   r   )r   r;   )r4   _r"   r"   r#   rd     s     z5TestConcatenate.test_concat_order.<locals>.<listcomp>d   Tr   )r   r;   r   rJ   r   rL   rX   )r   dfsr!   rO   r"   r"   r#   test_concat_order  s
    z!TestConcatenate.test_concat_orderc                 C   sb   t tjddgdd}t tddg}t||gdd}t ddtdtdgtd}t|| d S )Nr(   rG   Int64r|   Tr~   )	r   r   r   r   r   r   r@   rL   r   )r   ro   rp   r!   rO   r"   r"   r#   .test_concat_different_extension_dtypes_upcasts  s
    z>TestConcatenate.test_concat_different_extension_dtypes_upcastsc                 C   sR   t ttdttdgddgd}t ttdttdd}t|| d S )Nr&   r%   FirstAnotherrE   )r   r   )r   r   r;   rL   r   )r   rO   r!   r"   r"   r#   test_concat_ordered_dict  s     z(TestConcatenate.test_concat_ordered_dictc              	   C   sv   t tjddddddgdgd}t tjddddddgd	gd}d
}tjt|d t||gdd W 5 Q R X d S )Nr)   r   r(   rG   r&   ro   r   r%   rp   z8Reindexing only valid with uniquely valued Index objectsrq   re   )r   r   r   r   rs   rt   r	   r   )r   r   rC   rv   r"   r"   r#   #test_concat_duplicate_indices_raise  s
    ""z3TestConcatenate.test_concat_duplicate_indices_raiseN)r   r   r   r$   rD   rP   r[   rs   markparametrizerh   rn   rw   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r"   r"   r"   r#   r   %   s0   2
)!	D	
 	r   dtfloatc                 C   sj   |t dj}|tjdg| |d|tjtjg| |d|tjdg| |dg}t|}|jj| ksftd S )Nr|   r(   r>   ndminr)   )	r@   ndimr   r   nanr   rK   r>   r   )r   Zframe_or_seriesdimsr   xr"   r"   r#   !test_concat_no_unnecessary_upcast  s    r   pdtr   c              	   C   sz   t ddf | j}|tjdg| |d|tjtjg|d|tjdg| |dg}t|}|jjdksltW 5 Q R X d S )NT)recordr(   r   )r   r)   float64)	r   r   r   r   r   r   rK   r>   r   )r   r   r   r   r   r"   r"   r#   test_concat_will_upcast  s    r   c                  C   sD   t ddgi} t dg i}t ddgi}t| |g}t|| d S )Nr*   r(   g      ?r   )r   rC   rO   r!   r"   r"   r#   0test_concat_empty_and_non_empty_frame_regression  s
    r   c                  C   s^   t tdddg} tddgddgddggdttjd}t| | gdd}t	
|| d S )Nr   r(   rG   datare   )r   r   r   astyper   SparseDtyper   r   r   rL   rM   )ro   rO   r!   r"   r"   r#   test_concat_sparse  s    r   c                  C   sr   t tjd } tt jdd g| d}tdgtd}tdd dgdddgd| }t	||gdd}t
|| d S )Nr(   r|   r   r   r   re   )r   r   r   r   r   r<   r   r   r   r   rL   r   )r>   ro   rp   rO   r!   r"   r"   r#   test_concat_dense_sparse(  s    r   rF   r   r.   c                 C   s   t dddgdddgd}tdd	d
gdd}tdddgdd}t|||gd| d}ddddgddd	dgddd
dgg}t| d df| d df| d df| d dfg}t ||d}t|| d S )Nr(   rG   r&   r%   r)   r   ro   rp         r   r   rU   r   r   r   r   r,   rF   r   ro   rp   rI   )r   r   r   r   from_tuplesrL   rM   )rF   rB   r   r    r!   expected_valuesexpected_columnsrO   r"   r"   r#   test_duplicate_keys2  s    "*r   c               	   C   s   ddg} t dddgdddgd}t||gd| d	}ddddgddddgddddgg}t| d
 df| d
 df| d df| d dfg}t ||d}t  tdt t|| W 5 Q R X d S )Nr   r(   rG   r&   r%   r)   r   r   r   r   ro   rp   rI   ignore)	r   r   r   r   r   r   r
   rL   rM   )rF   rB   r!   r   r   rO   r"   r"   r#   test_duplicate_keys_same_frameA  s    "*
r   objrx   r   rU   c                 C   s"   t | | g}t|t| std S r1   )r   r   typer   )r   r!   r"   r"   r#   test_concat_preserves_subclassQ  s    
r   c                  C   s   t dtjdddgddi} t dtddd	gi}t| |gd
d}t ddddddd	gidd}t|| t|| gd
d}t dddd	dddgidd}t|| d S )Nro   r(   rG   r&   r   r|   r%   r)   r   Tr~   )r   r   r   r   r   rL   rM   r   rC   r!   rO   r"   r"   r#   (test_concat_frame_axis0_extension_dtypes_  s    r   c                  C   s^   t ddgidd} t ddgidd}t| |gdd}t dd gd dgd	dd}t|| d S )
Nro   r   r   r|   rp   r(   Tr~   r   r   )Zdf_aZdf_br!   rO   r"   r"   r#   +test_concat_preserves_extension_int64_dtypem  s
    r   zdtype1,dtype2,expected_dtype)boolr   r   )booleanr   r   )r   r   r   )r   r   r   c                 C   sT   t ddg| d}t ddg|d}t||gdd}t ddddg|d}t|| d S )NTFr|   r~   )r   r   rL   r   )Zdtype1Zdtype2Zexpected_dtypeZser1Zser2r!   rO   r"   r"   r#   test_concat_bool_typesv  s
    r   )rF   	integrityZredr&   TFZbluec                 C   sl   t ddit ddit ddig}t|| |d}tt| dddg}t dddgt|d}t|| d S )	Nro   r(   rp   rG   r   r&   )rF   Zverify_integrityr+   )r   r   rf   zipr   r   rL   r   )rF   r   Zseries_listr!   ZtuplesrO   r"   r"   r#   test_concat_repeated_keys  s
    "r   c                  C   s   t jdgddd} tddgi| d}td gdd	d}td gtjgd
|d}t||gdd}td | d gtd}td d gtjtjgtjdgd|d}t	|| d S )Nz2021-04-08 21:21:14+00:00zdatetime64[ns, UTC]z
Time (UTC))r>   rV   rz   g!u?r   r@   zMaybe Time (UTC))rx   ry   rJ   re   r   r|   )rx   ry   rz   r+   )
r   ZDatetimeIndexr   r   r   r   r   r@   rL   rM   )Zdtir   idxr   r!   rN   rO   r"   r"   r#    test_concat_null_object_with_dti  s      r   c                  C   sl   t ddg} tddgg| d}tdgtdd}t||g}tddgtjtjgg| d}t	|| d S )Nr   )rz   r(   r(   rG   rI   r   r   )
r   r   r   r   Z
RangeIndexr   r   r   rL   rM   )mir   rC   r!   rO   r"   r"   r#   ,test_concat_multiindex_with_empty_rangeindex  s    r   r   r(   rG   r   Zcol1r|   c                 C   sF   |   }ddi|_|   }ddi|_t||g}t|jdksBtd S )Nr(   rG   r   )r-   attrsr   r   r   r   r   rC   rB   r"   r"   r#   test_concat_drop_attrs  s    

r  c                 C   sF   |   }ddi|_|   }ddi|_t||g}|jd dksBtd S )Nr(   )r-   r  r   r   r  r"   r"   r#   test_concat_retain_attrs  s    

r  df_dtyper   r   zdatetime64[ns]empty_dtyper@   c                 C   sp   t ddgddgd|d}t ddg| d}t||g}|}|dkr`| d	krV|d	}n
|d
}t|| d S )Nr(   rG   r*   r^   r|   r*   r^   )rJ   r>   r   r   r@   )r   r   r   rL   rM   r  r  rB   emptyr!   rO   r"   r"   r#   %test_concat_ignore_empty_object_float  s    
r
  c                 C   s   t ddgddgd|d}t tjgtjgd| d}t||gdd}|dkr^| dkrZd}nd	}t d ddgd ddgd|d}t|| d S )
Nr(   rG   r  r|   Tr~   r   r@   r   )r   r   r   r   rL   rM   r  r"   r"   r#   &test_concat_ignore_all_na_object_float  s    r  c                  C   sr   t dgtdgd} t ddgi}t| |j| jdgdd}t ddgtdtjgd }} t|| d S )	Nr(   z
2012-01-01r   ro   rG   rI   Tr~   )	r   r   r   r   ZreindexrJ   r   rL   rM   r   r"   r"   r#   %test_concat_ignore_empty_from_reindex  s
    "r  )Acollectionsr   r   r   decimalr   typingr   warningsr   r   Znumpyr   rs   Zpandas.errorsr	   r
   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r   r   r   r   r   r   r   Zpandas._testingZ_testingrL   Zpandas.core.arraysr   Zpandas.tests.extension.decimalr   r   r   r   Zsctypesr   r   r   r   r   r   r   ZSubclassedDataFramer   ZSubclassedSeriesr   r   r   r   r   r   r   r   r  r  Zskip_array_manager_invalid_testr
  r  r  r"   r"   r"   r#   <module>   s   $	   W
	



		
		
	 

 

