o
    &_h                     @   sZ  d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlZddlmZmZ ddlZddlZddlZddlmZ ddlZdd	lmZ eG d
d dZeG dd dZeG dd dZG dd dZG dd dZG dd dZG dd dZG dd dZ G dd dZ!dd Z"G dd dZ#d d! Z$e%d"kre$  dS dS )#zv
NSW License Checker
================================================================================================
    N)Path)	dataclass)ListDictOptional)datetime)ThreadPoolExecutoras_completed)fuzz)parserc                   @   s   e Zd ZU dZdZeed< dZeed< dZeed< dZ	eed	< d
Z
eed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dS )SimpleConfigzSimple configurationzAhttps://api.onegov.nsw.gov.au/oauth/client_credential/accesstoken	token_urlz8https://api.onegov.nsw.gov.au/securityregister/v1/verify
verify_urlzJBasic NmtISWVtWG8zR010VWFzR2JWNzE4Y2VUSW1kbkdvRms6blZYNWRVR05yVWc3d3NRMQ==basic_auth_header 6kHIemXo3GMtUasGbV718ceTImdnGoFkapi_keyU   name_similarity_required   max_retry_attempts<   request_timeout_secondsg      ?delay_between_requests   parallel_workersT	test_mode
   test_record_limitN)__name__
__module____qualname____doc__r   str__annotations__r   r   r   r   intr   r   r   floatr   r   boolr    r'   r'   /var/www/html/scrapers/nsw.pyr      s   
 r   c                   @   sT   e Zd ZU dZeed< eed< eed< eed< dZeed< dd	 Zd
e	fddZ
dS )EmployeeRecordz)Holds one employee's information from CSVpayroll_numberemployee_namelicense_number
csv_expiryr   csv_row_numberc                 C   sD   t | j | _t | j | _t | j | _t | j | _dS )z)Remove extra spaces and clean up the dataN)r"   r*   stripr+   r,   r-   selfr'   r'   r(   
clean_dataI   s   zEmployeeRecord.clean_datareturnc                 C   s   t | jo| jS )z9Check if this record has the minimum required information)r&   r,   r+   r0   r'   r'   r(   has_required_dataP      z EmployeeRecord.has_required_dataN)r   r   r    r!   r"   r#   r.   r$   r2   r&   r4   r'   r'   r'   r(   r)   @   s   
 r)   c                   @   s   e Zd ZU dZdZeed< dZeed< dZeed< dZ	eed< dZ
eed< dZeed	< d
Zeed< dZeed< dZeed< dZeed< defddZdS )SearchResultz%Holds the result of searching NSW API	Not Found
found_namelicense_typelicense_expirylicense_status
No Licensename_matchesexpiry_status error_messager   how_many_retries        search_time_secondsF
was_cachedr3   c                 C   s   | j dko| j S )z-Did we successfully find license information?r7   )r8   r@   r0   r'   r'   r(   is_successfulc   s   zSearchResult.is_successfulN)r   r   r    r!   r8   r"   r#   r9   r:   r;   r=   r>   r@   rA   r$   rC   r%   rD   r&   rE   r'   r'   r'   r(   r6   U   s   
 r6   c                   @   s\   e Zd ZdZdefddZdd Zdejfdd	Z	de
fd
dZdd Zdd Zdd ZdS )
APIManagerz)Manages API connections and rate limitingconfigc                 C   s0   || _ t | _g | _d| _t | _d | _	d S )NF)
rG   queueQueuesession_poolall_sessionssetup_complete	threadingLocklockaccess_token)r1   rG   r'   r'   r(   __init__o   s   


zAPIManager.__init__c                 C   s   | j rdS | jQ | j r	 W d   dS td| jj d |  s'tdt| jjD ]}|  }| j	
| | j| td|d  d q-d| _ td	 W d   dS 1 s\w   Y  dS )
z,Create HTTP sessions for parallel processingNzSetting up z API sessions...zFailed to get API access tokenz   Session    z readyTz/All API sessions ready for parallel processing!)rL   rO   printrG   r   _get_access_token	Exceptionrange_create_sessionrK   appendrJ   put)r1   isessionr'   r'   r(   setup_sessionsw   s"   
"zAPIManager.setup_sessionsr3   c                 C   s.   t  }|jddd| j | jjd |S )z-Create one HTTP session with optimal settingszNSW-License-Verifier/1.0application/jsonzBearer )z
User-Agentcontent-typeauthorizationapikey)requestsSessionheadersupdaterP   rG   r   r1   r[   r'   r'   r(   rW      s   
zAPIManager._create_sessionc              
   C   s   z3d| j jd}ddi}tj| j j||dd}|  | d| _| js-td W d	S td
 W dS  tj	yN } ztd|  W Y d}~d	S d}~ww )zGet access token from NSW APIr]   r^   r_   
grant_typeclient_credentials   rc   paramstimeoutrP   zERROR: No access token receivedFz"Successfully obtained access tokenTz#ERROR: Failed to get access token: N)
rG   r   ra   getr   raise_for_statusjsonrP   rS   RequestException)r1   rc   rk   responseer'   r'   r(   rT      s.   zAPIManager._get_access_tokenc                 C   s   | j s|   | j S )z)Get a session from the pool (thread-safe))rL   r\   rJ   rm   r0   r'   r'   r(   get_session   s   
zAPIManager.get_sessionc                 C   s   | j | dS )z Return session to pool when doneN)rJ   rY   re   r'   r'   r(   return_session   r5   zAPIManager.return_sessionc                 C   sN   d}| j D ]}z|  W q   |d7 }Y q|dkr%td| d dS dS )z$Close all sessions when program endsr   rR   z	WARNING: z" sessions failed to close properlyN)rK   closerS   )r1   failed_cleanupsr[   r'   r'   r(   cleanup_all_sessions   s   
zAPIManager.cleanup_all_sessionsN)r   r   r    r!   r   rQ   r\   ra   rb   rW   r&   rT   rs   rt   rw   r'   r'   r'   r(   rF   l   s    rF   c                   @   s   e Zd ZdZdedefddZdedefdd	Zdedefd
dZ	dededefddZ
dededefddZdedefddZdedefddZdejdedefddZdS )APILicenseSearcherz$Searches NSW API using HTTP requestsrG   api_managerc                 C   s&   || _ || _i | _t | _i | _d S N)rG   ry   search_cacherM   rN   
cache_lock
name_cache)r1   rG   ry   r'   r'   r(   rQ      s
   

zAPILicenseSearcher.__init__namer3   c                 C   s~   |sdS || j v r| j | S | |}|   }t|dkr2|d  dd|dd  }n|  }|| j |< |S )zEConvert name to standard format for comparison with enhanced cleaningr?   r   ,  N)r}   _clean_name_for_matchingr/   uppersplitlenjoin)r1   r~   cleaned_nameparts
normalizedr'   r'   r(   normalize_employee_name   s   


"
z*APILicenseSearcher.normalize_employee_namec                 C   s   |sdS t | }d| }|  }g }|D ] }t|dkr*t|dkr*q|dr6t|dkr6q|| qg }|D ]}d|v rQ|d}|| q@|| q@dd |D }|red|S |S )	z>Clean name by removing common variations and formatting issuesr?   r   rR   .r   -c                 S   s   g | ]}|  r|qS r'   )r/   ).0wordr'   r'   r(   
<listcomp>  s    z?APILicenseSearcher._clean_name_for_matching.<locals>.<listcomp>)	r"   r/   r   r   r   r   endswithrX   extend)r1   r~   cleanedwordsfiltered_wordsr   processed_wordshyphen_partsr'   r'   r(   r      s(   
z+APILicenseSearcher._clean_name_for_matchingcsv_nameapi_namec           	      C   s   |dkrdS |r
|sdS |  |}|  |}| |}| |}||kr&dS | | kr0dS t||t| | t| | t| | g}t|}|| j	j
kr^dS d|ddS )z<Compare names with enhanced cleaning and return match statusr7   zNo MatchYesNo (.1fz%))r   r   r   r
   token_set_ratiotoken_sort_ratiopartial_ratioratiomaxrG   r   )	r1   r   r   cleaned_csvcleaned_apinorm_csvnorm_apisimilarity_methods
similarityr'   r'   r(   check_name_similarity  s*   




z(APILicenseSearcher.check_name_similarityr-   
api_expiryc              
   C   s   |r|r|dkr
dS z;t j|dd}t j|dd}|t k }| | k}|r.|s.W dS |r5|r5W dS |s<|s<W dS |sC|rCW dS W d	S  tyb } zd
t|dd  W  Y d}~S d}~ww )z<Calculate expiry status comparison between CSV and API datesr7   r<   T)dayfirstActiveExpiredzActive - Date WrongzExpired - Date WrongUnknownError: N   )r   parser   todaydaterU   r"   )r1   r-   r   csv_dateapi_date
is_expireddates_matchrr   r'   r'   r(   calculate_expiry_statusE  s(   "z*APILicenseSearcher.calculate_expiry_statusemployeec                 C   s~   |j | jv r| j|j  }| |j|j|_d|_|S | |}| r=| j	 || j|j < W d   |S 1 s8w   Y  |S )z+Search for one employee's license using APITN)
r,   r{   r   r+   r8   r=   rD   _search_with_retriesrE   r|   )r1   r   cached_resultresultr'   r'   r(   search_single_license`  s   

z(APILicenseSearcher.search_single_licensec           
      C   s:  |j  st S d}t| jjD ]|}zO| j }z@t }| 	||}t | }|
 rQ| |j|j|_| |j|j|_||_||_|W | j| W   S W | j| n| j| w W q ty } zt|}|| jjd k r| jjd|  }	t|	 W Y d}~qd}~ww td| jj d| | jjdS )z(Try searching multiple times if it failsr?   rR   r   NzFailed after z attempts: )r@   rA   )r,   r/   r6   rV   rG   r   ry   rs   time_do_api_searchrE   r   r+   r8   r=   r   r-   r:   r>   rA   rC   rt   rU   r"   r   sleep)
r1   r   
last_errorattemptr[   
start_timer   search_timerr   	wait_timer'   r'   r(   r   w  sD   

 
z'APILicenseSearcher._search_with_retriesr[   c           	   
   C   sD  zhd|j  i}|j| jj|| jjd}|jdkrE| }t|t	rA|rA|d }t
|dd|dd|dd|d	dd
W S t
 W S |jdkrNtd|jdkrb|jdd}td| d|  W dS  tjyx   td|j   tjy } z	tdt| d}~w ty } z	tdt| d}~ww )z/Actually search the NSW API using HTTP requestslicenceNumber)rk   rl      r   licenceNamer7   licenceType
expiryDatestatus)r8   r9   r:   r;   i  zAPI rate limit exceeded (408)i  zRetry-After2z Rate limited (429), retry after szAPI timeout for license: zAPI request error: NzAPI search error: )r,   r/   rm   rG   r   r   status_codero   
isinstancelistr6   rU   rc   rn   ra   Timeoutrp   r"   )	r1   r[   r   rk   rq   datar   retry_afterrr   r'   r'   r(   r     sB   






z!APILicenseSearcher._do_api_searchN)r   r   r    r!   r   rF   rQ   r"   r   r   r   r   r)   r6   r   r   ra   rb   r   r'   r'   r'   r(   rx      s    )(+rx   c                   @   s.   e Zd ZdZedededee fddZ	dS )SimpleCSVHandlerz"Reads employee data from CSV files	file_pathrG   r3   c              
      s  zt j| tdd  jj jdd _g d} fdd|D }|r5tdd	| d
t j dg } 	 D ]9\}}t
|d |d |d |d |d d}|  | rt|| |jrtt||jkrttd|j d  nq;|s{tdg }t }	d}
|D ]}|j|	vr|	|j || q|
d7 }
q|
dkrtd|
 d |W S  ty } z
tdt|   d}~ww )z Load employee data from CSV fileF)dtype	na_filteru   ﻿r?   )Payroll NumberEmployee NameLicense NumberExpiry/Update  Datec                    s   g | ]	}| j vr|qS r'   )columns)r   coldfr'   r(   r     s    z7SimpleCSVHandler.load_employee_data.<locals>.<listcomp>z"
ERROR: Missing required columns: r   z

Your CSV file MUST have these exact column names:
- Payroll Number
- Employee Name
- License Number
- Expiry/Update  Date

Current columns in your file: z
                r   r   r   r   r   r*   r+   r,   r-   r.   z!TEST MODE: Processing only first  recordsz2ERROR: No valid employee records found in CSV filer   rR   zNOTE: Removed z duplicate license numberszERROR reading CSV file: N)pdread_csvr"   r   r/   replace
ValueErrorr   r   iterrowsr)   r2   r4   rX   r   r   r   rS   setr,   addrU   )r   rG   required_columnsmissing_columns	employeesindexrowr   unique_employeesseen_licensesduplicates_removedrr   r'   r   r(   load_employee_data  sZ   	



z#SimpleCSVHandler.load_employee_dataN)
r   r   r    r!   staticmethodr"   r   r   r)   r   r'   r'   r'   r(   r          r   c                   @   st   e Zd ZdZdefddZdd Zdedefd	d
Z	dde
dedefddZde
defddZde
defddZdS )SimpleProgressTrackerz1Shows nice progress bar and processing statisticstotal_employeesc                 C   s<   || _ d| _d| _d| _d| _t | _d| _t	 | _
d S )Nr   )r   	completed
successfulfailedcachedr   r   last_updaterM   rN   rO   )r1   r   r'   r'   r(   rQ      s   
zSimpleProgressTracker.__init__c              
   C   sd   t dd  t d| j d t d  t ddddd	dd
dddddd 	 t d  dS )zShow the processing header
U=====================================================================================z!NSW LICENSE CHECKER - PROCESSING z
 EMPLOYEESProgress<15r   Employee<25LicenseStatus<12SpeedzU-------------------------------------------------------------------------------------N)rS   r   r0   r'   r'   r(   show_header*  s
   
,z!SimpleProgressTracker.show_headerr   r   c                 C   sl  | j - |  jd7  _|jr|  jd7  _n| r"|  jd7  _n|  jd7  _W d   n1 s3w   Y  t }|| j dk rEdS || _| j| j	 d }| 
|}t|| j d}| j| d }t|jdkrt|jdd d	 n|j}t|jd
kr|jdd d	 n|j}	|jrd}
n	| rd}
nd}
td| d|dd|	dd|
dd|ddddd dS )zUpdate progress displayrR   Ng      ?d   g{Gz?r         z..      CachedSuccessFailedr   r   r   r  .0fz/minr?   T)endflush)rO   r   rD   r   rE   r   r   r   r   r   _make_progress_barr   r   r   r+   r,   rS   )r1   r   r   current_timepercentprogress_barelapsedspeeddisplay_namedisplay_licenser   r'   r'   r(   update_progress2  sN   
.
z%SimpleProgressTracker.update_progressr   r  widthr3   c                 C   s8   t || d }d| d||   }d| d|ddS )zCreate ASCII progress barr  Xr   [z] z5.1f%r$   )r1   r  r  filledbarr'   r'   r(   r  ^  s   z(SimpleProgressTracker._make_progress_barelapsed_timeexcel_file_pathc                 C   s   t dd  t d t d  t d| j  t d| j  t d| j  t d| j  t d| jt| jd	 d
 dd t d| |  t d| j| d dd t d|  t d  dS )zShow final processing summaryz

r   zPROCESSING COMPLETED!zTotal employees processed: zSuccessful searches: zFailed searches: zCached results: zSuccess rate: rR   r  r   r  zTotal time: zProcessing speed: r   z employees/minutezExcel report saved: N)rS   r   r   r   r   r   _format_time)r1   r   r!  r'   r'   r(   show_final_summaryd  s   
$z(SimpleProgressTracker.show_final_summarysecondsc                 C   sp   |dk r
|ddS |dk r"t |d }t |d }| d| dS t |d }t |d d }| d| dS )	z"Convert seconds to readable formatr   r  z secondsi  zm r   zh mr  )r1   r$  minutessecshoursr'   r'   r(   r"  s  s   z"SimpleProgressTracker._format_timeN)r   )r   r   r    r!   r$   rQ   r  r)   r6   r  r%   r"   r  r#  r"  r'   r'   r'   r(   r     s    
,r   c                   @   s.   e Zd ZdZedee dedefddZdS )SimpleExcelGeneratorz'Creates Excel reports with color codingresultsoriginal_file_pathr3   c                 C   s  zPt |}t d}|j|j d| d }g }| D ]e}|dd}|dd}|dd}	|dd}
|d	d}|d
kpz|dkpz|dkpz|dpz|d
kpz|dkpz|	d
kpz|	dkpz|
dkpz|
dkpz|
dpzd|
v pzd|
v pz|d
kpz|dk}|r|| q|sdddt	|  dddddddd
}|g}|j|j d| d }t
|}t
j|dd}t	|dksdt|d  d!dvrd"nd#}|j|d$|d% |j}|j| }|d&d'd(dd)d*d&d+}t|jD ]6\}}|d ||| tt	t||js|| tj	  nd }tt|d, d-d.}|||| q|d d/ |d d t	|t	|jd  |dd W d0   n	1 sHw   Y  t|W S  tyh } z
td1t|   d0}~ww )2zDCreate Excel report - EXCEPTIONS ONLY (problems that need attention)z%Y%m%d_%H%M%S_Exceptions_z.xlsx
Name Matchr?   Database NameTypeExpiry Statusr   r7   r<   Errorr   r   zError:z
Date Wrongr   zN/Az"ALL RECORDS PROCESSED SUCCESSFULLYr   zNo exceptionsz	All foundzAll dates matchz	All validz
All active)
r   r   Rolecall Namer.  r-  r/  Rolecall ExpiryDatabase Expiryr0  r   _NSW_ALL_SUCCESS_
xlsxwriter)enginerR   zALL RECORDSr   r2  
ExceptionszProcessing SummaryF)r   
sheet_nameTz#8C1E31whitecentervcenter)boldbg_color
font_colorborderalignvalign	text_wrapr      2   r   NzERROR creating Excel report: )r   r   nowstrftimeparentstemrm   
startswithrX   r   r   	DataFrameExcelWriterr"   to_excelbooksheets
add_format	enumerater   writer   emptyastypemin
set_columnset_row
autofilterfreeze_panesrU   rS   )r*  r+  original_path	timestamp
excel_pathexception_resultsr   
name_matchdatabase_namer9   r>   r   is_exceptionsummary_resultr   writerr9  workbook	worksheetheader_colorcol_numheader
max_length	col_widthrr   r'   r'   r(   create_excel_report  s   	





"
)z(SimpleExcelGenerator.create_excel_reportN)	r   r   r    r!   r   r   r   r"   rj  r'   r'   r'   r(   r)    r   r)  c                   @   st   e Zd ZdZdd Zdd Zdd Zdefd	d
Zde	de
fddZdee dee fddZdedefddZdS )NSWLicenseCheckerz&Main application class - using NSW APIc                 C   s   t  | _t| j| _d | _d S rz   )r   rG   rF   ry   progress_trackerr0   r'   r'   r(   rQ     s   
zNSWLicenseChecker.__init__c              
   C   s4  zz?|    |  }t|| j}| t|s%td W W | j	  dS | 
|}t||}t | jj }| j|| W n; tyM   td Y n7 ty{ } z#tdt|  td td td td td	 W Y d}~nd}~ww W | j	  dS W | j	  dS W | j	  dS | j	  w )
z0Main function - this is where everything happenszProcess cancelled by userNz*

Process stopped by user (Ctrl+C pressed)z
ERROR: z
TROUBLESHOOTING TIPS:z1. Make sure your CSV file has the exact column names: 'Payroll Number', 'Employee Name', 'License Number', 'Expiry/Update  Date'!2. Check your internet connection.3. API credentials might be expired or invalidB4. Try reducing parallel_workers to 1 if getting rate limit errors)_show_welcome_get_csv_file_from_userr   r   rG   _ask_user_to_continuer   rS   ry   rw   _process_all_employeesr)  rj  r   rl  r   r#  KeyboardInterruptrU   r"   )r1   csv_file_pathr   r*  r\  r   rr   r'   r'   r(   run   s>   

zNSWLicenseChecker.runc                 C   s   t d dS )zShow welcome messagez!NSW License Checker - Starting...N)rS   r0   r'   r'   r(   rp  (  s   zNSWLicenseChecker._show_welcomer3   c                 C   s   t tjdkrtjd  d}td|  ntd td td td td td	 d}|s<td
t| sItd| tdt|j	  |S )zGet CSV file path from userrR   z"'zUsing file from command line: z
Please provide your CSV file:z   You can either:z   1. Type the full file pathz-   2. Drag and drop the file into this windowz"   3. Copy and paste the file pathz
Enter CSV file path: zNo file path providedzFile not found: zFile found: )
r   sysargvr/   rS   inputrU   r   existsr~   )r1   r   r'   r'   r(   rq  ,  s   z)NSWLicenseChecker._get_csv_file_from_useremployee_countc                 C   s~   | j jrtdt|| j j d dS td| d td| j j d 	 td  }|dv r4dS |d	v r:d
S td q&)z.Ask user if they want to process the employeesz
TEST MODE: Will process z
 employeesTz
Ready to process z	Will use z parallel API sessionsz"
Continue with processing? (Y/N): )yyes)nnoFz&Please enter 'y' for yes or 'n' for no)	rG   r   rS   rU  r   r   ry  r/   lower)r1   r{  rq   r'   r'   r(   rr  G  s   z'NSWLicenseChecker._ask_user_to_continuer   c                 C   s  t t|| _t| j| j}g }t| jjd}i }t|D ]\}}t	
d ||j|}|||< qt|D ]y}|| }z0| }	|j|j|j|	j|	j|j|	j|	j|	j|	jd
}
||
 | j||	 t	
| jj W q7 ty } z7td|j dt|  |j|j|jdd|jddddd
}|| tt|d}| j|| W Y d}~q7d}~ww W d   |S 1 sw   Y  |S )	zAProcess all employees with minimal output unless there are issues)max_workersg?)
r   r   r2  r.  r-  r3  r4  r0  r/  r   zERROR processing z: r1  )r@   N) r   r   rl  rx   rG   ry   r   r   rQ  r   r   submitr   r	   r   r*   r,   r+   r8   r=   r-   r:   r>   r9   r;   rX   r  r   rU   rS   r"   r6   )r1   r   searcherall_resultsexecutorfuture_to_employeerZ   r   futuresearch_resultresult_dictrr   error_resulterror_search_resultr'   r'   r(   rs  Z  sd   




@@z(NSWLicenseChecker._process_all_employeesr   c                 C   s   g }|j r
|d |jdkr|d|jdd |jdkr(|d|j  |jr5|d|j  n|d |rAd	|S d
S )z-Create informative notes for the Excel reportz!Cached result (duplicate license)r   zSearch time: z.2fr   z	Retries: r   zAPI requestz | zProcessed successfully)rD   rX   rC   rA   r@   r   )r1   r   notesr'   r'   r(   _create_notes_for_result  s   



z*NSWLicenseChecker._create_notes_for_resultN)r   r   r    r!   rQ   rv  rp  r"   rq  r$   r&   rr  r   r)   r   rs  r6   r  r'   r'   r'   r(   rk    s    (Mrk  c               
   C   s   z2t  } d| jd}ddi}tj| j||dd}|  | d}|r,td W d	S td
 W dS  tyb } z$tdt	|  td td td td td W Y d }~dS d }~ww )Nr]   rf   rg   rh   ri   rj   rP   zAPI connection test PASSED!Tz0API token test failed - no access token receivedFzAPI connection test failed: zCommon fixes:z1. Check internet connectionz#2. API credentials might be expiredz 3. NSW API service might be downz"4. Try again later if rate limited)
r   r   ra   rm   r   rn   ro   rS   rU   r"   )rG   rc   rk   rq   rP   rr   r'   r'   r(   test_api_connection  s:   r  c                   @   s^   e Zd ZdZdd Zdedee fddZdee defd	d
Z	dee dee fddZ
dS )NSWLicenseCheckerAPIz.API-compatible wrapper for NSW License Checkerc                 C   s(   t  | _t| j| _t| j| j| _d S rz   )r   rG   rF   ry   rx   r  r0   r'   r'   r(   rQ     s   zNSWLicenseCheckerAPI.__init__r   r3   c           	      C   s>  zzt || j}| j  g }|D ]h}z8| j|}|j|j|j	|j
|j|j|j|j|j|j|jr5|jnd|j|j|jd}|| t| jj W q tyz } z"|j|j|j	dd|jddddt|dddd}|| W Y d}~qd}~ww |W W | j  S  ty } z	tdt| d}~ww | j  w )z;Process CSV file and return results as list of dictionariesNr   r   r2  r.  r-  r3  r4  r0  r/  r   zError MessagezSearch TimeRetries
Was Cachedr1  rB   r   FNSW processing failed: )r   r   rG   ry   r\   r  r   r*   r,   r+   r8   r=   r-   r:   r>   r9   r;   r@   rC   rA   rD   rX   r   r   r   rU   r"   rw   )	r1   r   r   r*  r   r  r  rr   r  r'   r'   r(   process_csv_file  sd   

z%NSWLicenseCheckerAPI.process_csv_filer*  c                 C   s   |si S t |}tdd |D }tdd |D }tdd |D }i }|D ]}|dd}||dd ||< q'i }	|D ]}|d	d}
|	|
dd |	|
< q>|||||dkr_|| d
 nd||	dS )z,Calculate processing statistics from resultsc                 s   s"    | ]}| d dvrdV  qdS r.  )r1  r7   rR   Nrm   r   rr'   r'   r(   	<genexpr>;       z<NSWLicenseCheckerAPI.get_processing_stats.<locals>.<genexpr>c                 s   s"    | ]}| d dv rdV  qdS r  r  r  r'   r'   r(   r  <  r  c                 s   s     | ]}| d drdV  qdS )r  FrR   Nr  r  r'   r'   r(   r  =  s    r-  r   r   rR   r0  r  )total_recordssuccessful_searchesfailed_searchescached_resultssuccess_ratename_match_breakdownexpiry_status_breakdown)r   sumrm   )r1   r*  totalr   r   r   r=   r   match_statusexpiry_statusesr>   r'   r'   r(   get_processing_stats5  s,   z)NSWLicenseCheckerAPI.get_processing_statsrecords_datac                 C   s  zzg }t |D ]8\}}tt|dd t|dd t|dd t|dd |d d}|  || q| j  g }|D ]h}z8| j	
|}|j|j|j|j|j|j|j|j|j|j|jrm|jnd|j|j|jd	}|| t| jj W qJ ty }	 z"|j|j|jd
d
|jd
d
d
d
t|	dddd	}
||
 W Y d}	~	qJd}	~	ww |W W | j  S  ty }	 z	tdt|	 d}	~	ww | j  w )zBProcess list of records and return results as list of dictionariesr   r?   r   r   r   rR   r   Nr  r1  rB   r   Fr  )rQ  r)   r"   rm   r/   r2   rX   ry   r\   r  r   r*   r,   r+   r8   r=   r-   r:   r>   r9   r;   r@   rC   rA   rD   r   r   rG   r   rU   rw   )r1   r  r   rZ   record_datar   r*  r  r  rr   r  r'   r'   r(   process_recordsU  sx   

z$NSWLicenseCheckerAPI.process_recordsN)r   r   r    r!   rQ   r"   r   r   r  r  r  r'   r'   r'   r(   r    s    D r  c               
   C   s   t tjdkrtjd  } | dv rt  dS ztd td t s+td td t }|  W dS  tyB   td Y dS  t	yu } z(td	t
|  td
 td td td td td W Y d}~dS d}~ww )zMain entry pointrR   )z--testz-ttestNzStarting NSW License Checker...z(
Testing API connection to NSW OneGov...z4Connection issues detected, but continuing anyway...zCIf all searches fail, the API credentials may be invalid or expiredz 
API application stopped by userz
FATAL ERROR: z
API TROUBLESHOOTING:z,1. Check your CSV file has the right columnsrm  rn  ro  z,5. Run with --test to check API connectivity)r   rw  rx  r  r  rS   rk  rv  rt  rU   r"   )argapprr   r'   r'   r(   main  s2   r  __main__)&r!   ro   r   pathlibr   dataclassesr   typingr   r   r   r   rw  concurrent.futuresr   r	   rM   rH   ra   	rapidfuzzr
   pandasr   dateutilr   r   r)   r6   rF   rx   r   r   r)  rk  r  r  r  r   r'   r'   r'   r(   <module>   sH    #g |Ohs I) @)
