a Of,@sddlmZddlmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZdd lmZdd lmZddlZdd lmZdd lmZmZdd lmZerddlmZGdddeZdS)) annotations) TYPE_CHECKINGN)using_pyarrow_string_dtype)lib)import_optional_dependency) ParserError ParserWarning)find_stack_level) pandas_dtype) is_integer) DataFrame)_arrow_dtype_mappingarrow_string_types_mapper) ParserBase) ReadBuffercspeZdZdZdddfdd Zdddd Zddd d Zd d d ddZddddZd dddZ Z S)ArrowParserWrapperz7 Wrapper for the pyarrow engine for read_csv() zReadBuffer[bytes]None)srcreturnc s$t|||_||_|dSN)super__init__kwdsr _parse_kwds)selfrr __class___/var/www/ai-form-bot/venv/lib/python3.9/site-packages/pandas/io/parsers/arrow_parser_wrapper.pyr%s zArrowParserWrapper.__init__rcCsN|jd}|durdn||_|jd}t|tr:tdt|jd|_dS)z? Validates keywords before passing to pyarrow. encodingNzutf-8 na_valuesz?The pyarrow engine doesn't support passing a dict for na_values)rgetr isinstancedict ValueErrorlistr!)rr r!rrrr,s   zArrowParserWrapper._parse_kwdscCsddddddd}|D]4\}}||jvr|j|dur|j||j|<q|j}t|trh|g}nd}||jd <d d |jD|_|jd }|durt|r||jd <nZ|t j j krd|jd <nB|t j j krdddd}||jd <n|t j j krdd|jd <dd |jD|_d|jdv|jd<|jdurld|jvrldd|jdD|jd<|jdu|jdur|jn|jd|jd|_dS)z: Rename some arguments to pass to pyarrow include_columns null_values escape_charignore_empty_lines decimal_point quote_char)usecolsr! escapecharZskip_blank_linesdecimal quotecharNtimestamp_parserscSs&i|]\}}|dur|dvr||qS)N) delimiterr,r)r*r.0Z option_nameZ option_valuerrr Ys z;ArrowParserWrapper._get_pyarrow_options.. on_bad_linesZinvalid_row_handlerstrrcSs.tjd|jd|jd|jttddS)Nz Expected z columns, but found z: ) stacklevelskip)warningswarnZexpected_columnsZactual_columnstextrr )Z invalid_rowrrrhandle_warningks z?ArrowParserWrapper._get_pyarrow_options..handle_warningcSsdS)Nr9r)_rrrvz9ArrowParserWrapper._get_pyarrow_options..cSs&i|]\}}|dur|dvr||qS)N)r'r(Z true_valuesZ false_valuesr+r1rr3rrrr5xs Zstrings_can_be_nullcSsg|]}d|qS)fr)r4nrrr sz;ArrowParserWrapper._get_pyarrow_options..Zskiprows)Zautogenerate_column_namesZ skip_rowsr )itemsrr"pop date_formatr#r7 parse_optionscallablerZBadLineHandleMethodERRORWARNZSKIPconvert_optionsheaderr read_options)rmappingZ pandas_nameZ pyarrow_namerGr6r=rrr_get_pyarrow_options:s\          z'ArrowParserWrapper._get_pyarrow_optionsr )framerc stj}d}|jdurn|jdur6|jdur6t||_t|j|krftt|t|j|j|_d}|j_|j\}|jdur|j}t |jD]\}}t |rj|||<n|jvrt d|d|j dur|j |dur||j |fnj||j j|f\}} | dur|| |<|j |=qj|ddd|jdur|sdgtjjj_|j durt|j trfdd|j D|_ n t|j |_ z|j Wn.ty} zt | WYd} ~ n d} ~ 00S) z Processes data read in based on kwargs. Parameters ---------- frame: DataFrame The DataFrame to process. Returns ------- DataFrame The processed DataFrame. TNFzIndex z invalid)ZdropZinplacecs$i|]\}}|jvr|t|qSr)columnsr )r4kvrQrrr5s z>ArrowParserWrapper._finalize_pandas_output..)lenrRrMnamesranger&Z_do_date_conversionsZ index_colcopy enumerater r%Zdtyper"ZastypeZ set_indexindexr#r$rEr TypeError) rrQZnum_colsZmulti_index_namedr>Z index_to_setiitemkeyZ new_dtypeerrUr_finalize_pandas_outputsP              z*ArrowParserWrapper._finalize_pandas_outputcCs:t|r&tdd|Ds&tdnt|r6tddS)Ncss|]}t|tVqdSrr#r7r4xrrr r@z7ArrowParserWrapper._validate_usecols..zwThe pyarrow engine does not allow 'usecols' to be integer column positions. Pass a list of string column names instead.z=The pyarrow engine does not allow 'usecols' to be a callable.)r is_list_likeallr%rI)rr-rrr_validate_usecolssz$ArrowParserWrapper._validate_usecolsc Cstd}td}|z|jfi|j}Wnhty|jdd}|dur\|||jdt}t |rt dd|DstdYn0z4|j |j |j fi|j|jfi|j|d }Wn0|jy}zt||WYd}~n d}~00|jd }|tjurj|j} |} t|jjD]0\} } |j| r.| | | | | } q.|| }|dkr|jtjd } nP|d krt }t!||"<|j|jd } n t#r|jt$d } n|} |%| S) z Reads the contents of a CSV file into a DataFrame and processes it according to the kwargs passed in the constructor. Returns ------- DataFrame The DataFrame created from the CSV file. Zpyarrowz pyarrow.csvr'Nr(css|]}t|tVqdSrrbrcrrrresz*ArrowParserWrapper.read..z9The 'pyarrow' engine requires all na_values to be strings)rNrHrL dtype_backend)Z types_mapperZnumpy_nullable)&rrPZConvertOptionsrLr\r"rhsetrrfrgZread_csvrZ ReadOptionsrNZ ParseOptionsrHZ ArrowInvalidrrZ no_defaultZschemaZfloat64rZtypesZis_nullfieldZ with_typecastZ to_pandaspdZ ArrowDtyper Z Int64Dtypenullrrra)rpaZ pyarrow_csvrLincludeZnullstabler`riZ new_schemanew_typer]Z arrow_typerQZ dtype_mappingrrrreads\          zArrowParserWrapper.read) __name__ __module__ __qualname____doc__rrrPrarhrt __classcell__rrrrr s[I r) __future__rtypingrr:Zpandas._configrZ pandas._libsrZpandas.compat._optionalrZ pandas.errorsrrZpandas.util._exceptionsr Zpandas.core.dtypes.commonr Zpandas.core.dtypes.inferencer Zpandasrnr Zpandas.io._utilr rZpandas.io.parsers.base_parserrZpandas._typingrrrrrrs