a
    Pf                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
Zddl
mZ ddlmZ e	jdZe	jje	jjgZedd	 Zd
d Zedd ZdS )zd
Tests multithreading behaviour for reading and
parsing files for each parser defined in parsers.py
    )	ExitStack)BytesIO)
ThreadPoolN)	DataFrameZpyarrow_xfailc           	         s   | }d d} fddt |D }t \fdd|D }td}||j|}|d }|D ]}t|| qdW d    n1 s0    Y  d S )	Nd   
   c                 3   s*   | ]"}d  dd t D  V  qdS )
c                 S   s&   g | ]}|d d|d d|d qS )d, .0ir   r   a/var/www/ai-form-bot/venv/lib/python3.9/site-packages/pandas/tests/io/parser/test_multi_thread.py
<listcomp>"       zBtest_multi_thread_string_io_read_csv.<locals>.<genexpr>.<listcomp>N)joinrangeencode)r   _)max_row_ranger   r   	<genexpr>!   s   z7test_multi_thread_string_io_read_csv.<locals>.<genexpr>c                    s   g | ]}  t|qS r   )enter_contextr   )r   b)stackr   r   r   (   r   z8test_multi_thread_string_io_read_csv.<locals>.<listcomp>   r   )r   r   r   r   mapread_csvtmassert_frame_equal)	all_parsersparserZ	num_filesZbytes_to_dffilespoolresultsZfirst_resultresultr   )r   r   r   $test_multi_thread_string_io_read_csv   s    
r&   c                    s   fdd} fddt D }td}|||}W d   n1 sR0    Y  |d j}|dd D ]
}	||	_qrt|}
|
S )	a  
    Generate a DataFrame via multi-thread.

    Parameters
    ----------
    parser : BaseParser
        The parser object to use for reading the data.
    path : str
        The location of the CSV file to read.
    num_rows : int
        The number of rows to read per task.
    num_tasks : int
        The number of tasks to use for reading this DataFrame.

    Returns
    -------
    df : DataFrame
    c                    sB   | \}}|s" j dd|dgdS  j ddt|d |dgdS )aj  
        Create a reader for part of the CSV.

        Parameters
        ----------
        arg : tuple
            A tuple of the following:

            * start : int
                The starting row to start for parsing CSV
            * nrows : int
                The number of rows to read.

        Returns
        -------
        df : DataFrame
        r   date)	index_colheadernrowsparse_datesN   	   )r(   r)   Zskiprowsr*   r+   )r   int)argstartr*   )r!   pathr   r   readerG   s    
z0_generate_multi_thread_dataframe.<locals>.readerc                    s    g | ]} |    fqS r   r   r   )num_rows	num_tasksr   r   r   i   s   z4_generate_multi_thread_dataframe.<locals>.<listcomp>)Z	processesNr   r,   )r   r   r   columnspdconcat)r!   r1   r3   r4   r2   tasksr#   r$   r)   rfinal_dataframer   )r3   r4   r!   r1   r    _generate_multi_thread_dataframe3   s    "*

r;   c                 C   s   d}d}| }d}t tjd|tjd|tjd|tjd|tjd|dg| dg| dg| tjd|d	d
tj|ddd
}t|4}|	| t
||||}t|| W d    n1 s0    Y  d S )N   0   z__thread_pool_reader__.csv   foobarbazz20000101 09:00:00s)ZperiodsfreqZint64)Zdtype)
ar   cr	   er?   r@   rA   r'   r.   )r   nprandomZdefault_rngr6   Z
date_rangeZaranger   Zensure_cleanZto_csvr;   r   )r    r4   r3   r!   	file_nameZdfr1   r:   r   r   r   )test_multi_thread_path_multipart_read_csvy   s.    
rJ   )__doc__
contextlibr   ior   Zmultiprocessing.poolr   numpyrG   ZpytestZpandasr6   r   Zpandas._testingZ_testingr   markZusefixturesZxfail_pyarrowZ
single_cpuZslowZ
pytestmarkr&   r;   rJ   r   r   r   r   <module>   s"   
F