o
    ҷhF                     @   s  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	m
Z
 ddlZddlZddlZddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZ ddlm  mZ ddlZdd	lm Z m!Z!m"Z" ddl#m$Z% dd
l&m'Z' ddl(m  m)Z* ddl+m,Z,m-Z- ej.dd Z/dd Z0ej12ddde3de4de5dedg de4d6dde4d6ddddd e7dj8ddde7dj8ddde7dj8ddddddddf gdd Z9dd  Z:d!d" Z;d#d$ Z<ej12d%ej=e;d&d'ej=ej>d(d'ej=ejej>ej?d)d*d'ej=ejej>d+d)d,d'ej=ejej>d-d)d.d'gej12d/ej>e;gd0d1 Z@d2d3 ZAd4d5 ZBej.d6d7 ZCG d8d9 d9ZDG d:d; d;ZEej12d<d=gd>e g d?fd@ejFGg d?g dAgfgdBdC ZHdDdE ZIej12dFg dGdHdI ZJdJdK ZKG dLdM dMejLZMdNdO ZNdPdQ ZOdRdS ZPdTdU ZQej12dVdWdXdVedYdZdZd[gd\d] ZRd^d_ ZSd`da ZTejUdbdc ZVej12ddejWej?gdedf ZXdgdh ZYdS )ia<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )arrayN)partial)Path)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)IndexSeriesperiod_rangecreate_pickle_data)DayMonthEndc                   C   s   t  S Nr    r   r   N/var/www/html/venv/lib/python3.10/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_data7   s   r   c                 C   s   t |trt||  d S |drt| | d S |dkr4|tju r,| tju s*J d S | |ks2J d S ttd| dtj	}|| | d S )Nsp_	timestampassert__equal)

isinstancer	   tmassert_index_equal
startswithassert_equalpdNaTgetattrassert_almost_equal)resultexpectedtyp
comparatorr   r   r   compare_element@   s   


r%   datas   123s   123456I)         B)r*   r)   r)      )r*      C)orderFc                 C   s   t | }t| d}||ksJ t| ttfr || u sJ d S t|trA|jdks,J |jdks3J |js8J |j	|j
fksCJ d S d S )NAr(   r+   )r   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r&   r!   r"   r   r   r   test_flatten_bufferV   s   

r;   c           
   	   C   s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkr|dkrt
|| |jj|jjksIJ |jjjrPJ t
|dk|dk |jj}|td tdksjJ |tjdd }	t|	tjszJ |	tjddd	ksJ |tjdd
 }	t|	tjsJ |	tjdddksJ q'|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d q'|dkr|dv rt
|| q'|dkr|dv rt
|| q't||| q'qqd S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*seriestsr   r(   r)   )hours)daysr>   )nanoseconds)r?   r@   indexperiodM)dt_tzcatframe)dt_mixed_tzs
cat_onecolcat_and_float)r   pytestskipr   __file__parentglobr   read_pickleitemsr   assert_series_equalrA   freq	normalizer   	Timedeltar   r   r   freqstrshiftassert_frame_equalr%   )
datapathlegacy_pickler&   r#   dvdtr!   r"   rR   resr   r   r   test_pickless   sF   

r]   c                 C   @   t |d}tj| |dd W d    d S 1 sw   Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler      "rl   c                 C   sB   t | d}|d t|W  d    S 1 sw   Y  d S )Nrbr   )re   seekrf   load)rj   rk   r   r   r   python_unpickler   s   
$rq   pickle_writerpython)idpandas_proto_defaultrb   pandas_proto_highestr-   pandas_proto_4   pandas_proto_5writerc                 C   s  | }|  D ]\}}|  D ]x\}}t h}||| t|}	t|	|| t|}	t|	|| t|dd}
||| |
d W d    n1 sLw   Y  t|dd}
t|
}	|
d W d    n1 slw   Y  t|	|| W d    n1 sw   Y  qqd S )Nr`   moder   rn   )	rP   r   ensure_cleanr   rO   r%   rq   re   ro   )r   rr   rz   r&   r#   rZ   r[   r"   rj   r!   handler   r   r   test_round_trip_current   s,   




r   c                  C   (   t  } t | jtj}t | | d S r   )r   makeDataFrameround_trip_pathlib	to_pickler   rO   rW   dfr!   r   r   r   test_pickle_path_pathlib      r   c                  C   r   r   )r   r   round_trip_localpathr   r   rO   rW   r   r   r   r   test_pickle_path_localpath   r   r   c                   C   s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s   r   c                   @   sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C   s  |d u rt || d S |dkrt|d}n|dkr"t|d}n|dkrJtj|dtjd}|	|t
j| W d    n1 sDw   Y  nb|dkrt|d/}tj|dd}||t
j|}||| W d    n1 svw   Y  W d    n1 sw   Y  n!|d	krt |d}n|d
krtd|d}n	d| }t||dvrt|d*}| |	|  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S d S )Ngzipwbz2zipcompressiontarrn   r{   xzzstd	zstandardr`   zUnrecognized compression type: )r   r   )shutilcopyfiler   re   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosrj   basenametarfile
gettarinfoaddfiler   r   
ValueErrorread)	selfsrc_path	dest_pathr   frk   r   tarinfomsgr   r   r   compress_file   sH   
"zTestCompression.compress_filec                 C   s  |}|d }|d }t |q}t |S}t  }|j||d t j||d$}	t|d}
|
|	  W d    n1 s@w   Y  W d    n1 sOw   Y  tj	|d d}t 
|| W d    n1 skw   Y  W d    d S W d    d S 1 sw   Y  d S )N.compressed.rawr   r`   )r   r}   r   r   decompress_filere   r   r   r   rO   rW   )r   r   r   basepath1path2p1p2r   r   rk   df2r   r   r   test_write_explicit  s   Pz#TestCompression.test_write_explicitr   ) Nonebad7zc              	   C   s   t jtdd1 t|}t }|j||d W d    n1 s#w   Y  W d    d S W d    d S 1 s;w   Y  d S )NzUnrecognized compression type)matchr   )rJ   raisesr   r   r}   r   r   )r   r   r   rj   r   r   r   r   test_write_explicit_bad&  s   "z'TestCompression.test_write_explicit_badc                 C   s   |}|| }|d }| j | }t|o}t|Q}t }	|	| tj||d$}
t|d}|	|

  W d    n1 sFw   Y  W d    n1 sUw   Y  tj|d d}t|	| W d    n1 sqw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r`   )_extension_to_compressiongetlowerr   r}   r   r   r   re   r   r   r   rO   rW   )r   compression_extr   r   r   r   r   r   r   r   r   rk   r   r   r   r   test_write_infer-  s    
Pz TestCompression.test_write_inferc           
   	   C   s   |}|d }|d }t |F}t |(}t  }|j|d d | j|||d tj||d}	t ||	 W d    n1 s@w   Y  W d    d S W d    d S 1 sXw   Y  d S )Nr   r   r   )r   r}   r   r   r   r   rO   rW   )
r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_explicitC  s   Pz"TestCompression.test_read_explicitc              	   C   s   |}|d }|| }| j | }t|D}t|&}t }	|	j|d d | j|||d t	|}
t
|	|
 W d    n1 sFw   Y  W d    d S W d    d S 1 s^w   Y  d S )Nr   r   )r   r   r   r   r}   r   r   r   r   rO   rW   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_inferU  s   
PzTestCompression.test_read_inferN)__name__
__module____qualname__icomextension_to_compressionr   r   r   rJ   markparametrizer   r   r   r   r   r   r   r   r      s    
r   c                   @   s&   e Zd Zejdg ddd ZdS )TestProtocolrc   )ra   r   r(   r)   c                 C   s\   t |}t  }|j||d t|}t || W d    d S 1 s'w   Y  d S )Nrb   )r   r}   r   r   r   rO   rW   )r   rc   r   rj   r   r   r   r   r   	test_reado  s   
"zTestProtocol.test_readN)r   r   r   rJ   r   r   r   r   r   r   r   r   n  s    r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)r1   r+   r.   c                 C   s*   | ddd|}t |}t|j| d S )Nior&   rf   )r   rO   r   r   columns)rX   r   r   rj   r   r   r   r   test_unicode_decode_errorx  s   
r   c               	   C   s   t  G} t  }t| d}|| W d    n1 sw   Y  t| d}t|}W d    n1 s8w   Y  t || W d    d S 1 sNw   Y  d S )Nr`   rn   )r   r}   r   re   r   r   rO   rW   )rj   r   rk   r!   r   r   r   test_pickle_buffer_roundtrip  s   
"r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                    s   dd }G dd d t  * fdd}t  }|| | d| t|}t || W d    d S 1 s<w   Y  d S )Nc                 S   r^   r_   rd   rh   r   r   r   rl     rm   z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s6   e Zd ZdddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsereturnNc                 S   s0   t |d| _d|v rddi| _d S ddi| _d S )Nrn   r   zContent-Encodingr   )re   fileheaders)r   rj   r   r   r   __init__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r   r   r   r   r   r   	__enter__     z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r   )close)r   argsr   r   r   __exit__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   
   | j  S r   )r   r   r   r   r   r   r        
z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   r   r   )r   r   r   r   r   r   r     r   z;test_pickle_generalurl_read.<locals>.MockReadResponse.closer   N)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s    
r   c                     s    S r   r   )r   kwargsr   rj   r   r   mock_urlopen_read  s   z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   r}   r   setattrr   rO   rW   )monkeypatchr   rl   r   r   r!   r   r   r   test_pickle_generalurl_read  s   


"r   c                  C   sd   t d t  d} t }||  t| }t|| W d    d S 1 s+w   Y  d S )Nfsspeczmemory://mockfile)	rJ   importorskipr   r}   r   r   r   rO   rW   )r   r   r!   r   r   r   test_pickle_fsspec_roundtrip  s   



"r   c                   @   s   e Zd ZdddZdS )MyTzr   Nc                 C   s   d S r   r   r   r   r   r   r     r   zMyTz.__init__r   )r   r   r   r   r   r   r   r   r     s    r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts J d S )N)dtyper   r(   )r
   objectr   r   round_trip_picklerQ   r   )r"   r!   r   r   r   test_read_pickle_with_subclass  s   
r   c                 C   s   t  }t  }|j|| d t| }W d   n1 s w   Y  t }|j|| d |d |	 |ksA| dv sAJ t
j|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r   Nr   )r   r   r   )r   r   r}   r   r   
read_bytesr   BytesIOro   getvaluer   rO   rW   )r   r   rj   	referencebufferread_dfr   r   r   %test_pickle_binary_object_compression  s   


r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r   )r   r   rW   )rF   	unpickledr   r   r   _test_roundtrip     
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)/multiindex_year_month_day_dataframe_random_data multiindex_dataframe_random_dataymdrF   r   r   r   r   +test_pickle_dataframe_with_multilevel_index  s   
r  c                  C   sH   t dddd} ttjdt| | }t|}|j	j
dks"J d S )Nz1/1/2011z1/1/2012rC   )rR   r)   )r   r
   nprandomdefault_rngstandard_normallenr   r   rA   rR   )prngr=   new_tsr   r   r   "test_pickle_timeseries_periodindex  s   
r  namei	  g     H@i     )r(   r)   c                 C   s$   t t j| d}|j| ksJ d S )N)r  )r   r   makeTimeSeriesr  )r  r   r   r   r   test_pickle_preserve_name  s   r  c                 C      t | }t ||  d S r   r   r   rQ   )datetime_seriesunp_tsr   r   r   test_pickle_datetimes  r   r  c                 C   r  r   r  )string_series
unp_seriesr   r   r   test_pickle_strings!  r   r  c                  C   sf   t tddjdg } t| }|jjd jdksJ |jjd j	dks(J t
|dg |  d S )Nabccategoryr   r(   )r(   T)r
   listastypeilocr   r   _mgrblocksr6   r9   rQ   )serr\   r   r   r    test_pickle_preserves_block_ndim&  s
   
r   rc   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rc   r   r   )	r   	DataFrameranger   r   r   r   rO   rW   )rc   r   r   r!   r   r   r   %test_pickle_big_dataframe_compression3  s   r#  c                 C   sh   | t tjdddd}t|d}t|}W d    n1 s w   Y  tjg g d}t	|| d S )Nr&   rY   z1.2.4zempty_frame_v1_2_4-GH#42345.pklrn   )rA   r   )
r   rL   rM   re   rf   rp   r   r!  r   rW   )rX   rj   fdr   r"   r   r   r   #test_pickle_frame_v124_unpickle_130>  s   r%  )Z__doc__r   r   datetime	functoolsr   r   r   r   pathlibr   rf   r   r   r   r   numpyr  rJ   pandas.compatr   r   pandas.compat._optionalr   pandas.compat.compressorsr   pandas.util._test_decoratorsutil_test_decoratorstdpandasr   r	   r
   r   pandas._testing_testingr   -pandas.tests.io.generate_legacy_storage_filesr   pandas.io.commoncommonr   pandas.tseries.offsetsr   r   fixturer   r%   r   r   r5   r2   PickleBuffercastarangereshaper;   r]   rl   rq   paramr   HIGHEST_PROTOCOLr   r   r   r   r   r   
MultiIndexfrom_arraysr   r   r   r   tzinfor   r   r   r  r  r  r  r  skip_array_manager_invalid_testr   DEFAULT_PROTOCOLr#  r%  r   r   r   r   <module>   s    
$
2
~



%
	



