U
    ðQÒh.—  ã                   @   s  d dl Z d dlZd dlmZmZmZmZ d dlmZ d dl	Z	d dl
mZ d dlmZ d dlZd dlmZ d dlZd dlmZmZmZmZmZ d dlZd dlm  mZ d dlm  mZ d dl m  m!Z" d dl#mZ e $¡ Z%e &d	¡Z'd
Z(eƒ a)ej*e( Z+e j, -e+d¡Z.e j, -e+d¡Z/e j, -e+d¡Z0e 1¡ a2ddddœZ3eƒ Z4e4j5ee3d dd„ Z6dd„ Z7dd„ Z8et)ddd„ ƒZ9dd„ Z:dd„ Z;d d!„ Z<d"d#„ Z=d$d%„ Z>d&d'„ Z?d(d)„ Z@d*d+„ ZAd,d-„ ZBd.d/„ ZCd0d1„ ZDd2d3„ ZEe4 F¡ d4d5„ ƒZGd6d7„ ZHd8d9„ ZIe;ƒ  dS ):é    N)ÚdatetimeÚdateÚtimeÚ	timedelta)Útz)ÚCache)Úprofile)ÚStringIO)ÚhtmlÚdccÚcallbackÚ
dash_tableÚclientside_callback)ÚappzAustralia/MelbourneÚaemeryúdata_stock.csvúdata_orders.csvúdata_po.csvTZSimpleCachei€Q )ÚDEBUGZ
CACHE_TYPEZCACHE_DEFAULT_TIMEOUT)Úconfigc                 C   s4   ||kr ||   ko|k S   S || kp.| |k S d S ©N© )ÚnowÚstartÚendr   r   ú,/var/www/FlaskApp/FlaskApp/app/data_store.pyÚ
in_between3   s    r   c                 C   s(   || kr| |   |¡ | | S |gS d S r   )Úappend)ÚdÚkÚvr   r   r   Údict_append9   s    r!   c                 C   s>   g }t | ƒ d¡}|D ]}tj|df }| |¡ qd |¡S )Nú,Únamez, )ÚstrÚsplitÚ	season_dfÚlocr   Újoin)Zs_idÚseasonsZs_listÚsÚseasonr   r   r   Údecode_season_id@   s    r,   )Ústreamc                 C   sX   t j}tt _tdt|ƒ d | d t|ƒ d t|ƒ ƒ t | ¡}tdƒ |t _|S )NzMemory data at z for z iteration no. Ú zEnd of Data)ÚsysÚstdoutÚmem_analysisÚprintr$   ÚpdÚconcat)Údfsr   ÚtextÚ	iterationÚ
old_stdoutÚ	return_dfr   r   r   Úpd_concat_with_memJ   s    0
r:   c           0      C   s®  zZt j d¡ t j|  }tj |d¡}tj |d¡}tj |d¡}t  d| d¡}t jrbt	 
¡ }nt	 
¡ jtd t¡jd d}t| ¡ tdd	ƒtd
ƒƒròd}t j| d|dd}|d rÐt j d|  ¡ d an8|d }	tj |	d ¡atd  t¡td< tjddd t  | |¡}
|
r<tj|
dddddtttdœd}nt ¡ }d}d}|rt j dt|ƒ ¡ t j| d|d t|ƒ dd}t j dt|d ƒ ¡ |d r¾t j d|  ¡ qqL|d }	i }|	d D ]}|d  D ]}|d! D ]ð}|d" rît|d#|ƒ|d#< t|d$|d ƒ|d$< t|d%|d& ƒ|d%< t|d'|d( ƒ|d'< t|d)|d* ƒ|d)< t|d+d, d-d.„ |d+ D ƒ¡ƒ|d+< t|d/|d/ ƒ|d/< t|d0|d0 ƒ|d0< t|d1|d1 ƒ|d1< t|d2|d ƒ|d2< t|d"|d" ƒ|d"< t|d3|d3 ƒ|d3< t|d4|d4 ƒ|d4< t|d5|d5 ƒ|d5< t|d6|d6 ƒ|d6< t|d7|d7 ƒ|d7< |d8 D ]Š}t|d9|d(  d: |d; ƒ|d9|d(  d: < t|d9|d(  d< |d= ƒ|d9|d(  d< < t|d9|d(  d> |d? ƒ|d9|d(  d> < qRqîqàqÒt|tj |¡g|d@|ƒ}|	dA dB }qLt  ddCt  ¡ dD¡ td k	rH|j!dEdF„ ddG|dH< t "|d# ¡|dI< |dI j#j$|dJ< ||dI  j#j%|dK< ||dJ d
k|dK dLk@ |dK dLkB  }|j&dIdJdKgddM |j'dddN}t j(| ||ddO t j )dP|  dQ | ¡ dRdSd"dTdUdVdWdXdYdZd[g}d\d]d"d^g}t  | |¡}
|
rNtj|
dddddd_d_ttd`œd}ntj|da}t  | |¡}
|
r†tj|
dddtdbœd}ntj|da}t j*| tj |dc¡t	 
¡ t+ddde dWgdf}|t j*| tj |dg¡t	 
¡ t+ddde d[dhgdf }|
r tjdRdSd"dTdUdVdWgda}tjdRd"dXdYdZd[gda}|D ]Ð}t  di|dj ¡}
|
	rÞ|
 ,¡  -dk¡ .dl¡}t/ 0|dmdn¡}|dWkr²t/ 0|dodp¡}|dqk	rÜt/ 0|dodr¡} t/ 0|dodn¡}!t/ 0|dod
¡}"t/ 0|dsdt¡}#t1|#ƒtkrØ|#g}#t/ 0|dsdu¡}$t1|$ƒtkrú|$g}$|#	rÜt2t3|#ƒƒD ] }%i }|dv jtd t¡jd dg|dT< |!g|dR< |"g|dS< | g|dU< |#|% g|d"< |$|% g|dV< dg|dW< t 4|tj |¡g¡}|j5dRdUd"dTgdddw q	qü|d[krÚt/ 0|dxdp¡}!t/ 0|dydp¡}#t1|#ƒtkrì|#g}#t/ 0|dydt¡}&t1|&ƒtkr|&g}&t/ 0|dydu¡}'t1|'ƒtkr0|'g}'t2t3|#ƒƒD ]˜}%i }|!g|dR< |dv jtd t¡jd dg|dX< |#|% g|d"< |&|% g|dY< |'|% g|dZ< dg|d[< t 4|tj |¡g¡}|j5dRd"dXgdddw q<	qü|dhk	rüt/ 0|dhdp¡}(t1|(ƒtk	r|(g}(t/ 0|dhdu¡}#t1|#ƒtk	r(|#g}#t/ 0|dhd
¡})t1|)ƒtk	rJ|)g})t2t3|#ƒƒD ]„}%i }|(|% g|d\< |dv jtd t¡jd dg|d]< |#|% g|d"< |)|% g|d^< t 4|tj |¡g¡}|j5d\d"d]gdddw 	qVnt j dzt|ƒ d{ ¡  
q q,|j6|dRd"gd|d}}*t3|*j7ƒdk
r t 4||*g¡}|dS jjd~ddd€|dS< |j5dRdUd"dTdXgdddw |j8 9¡ }+|j&dd‚„ |+D ƒddƒd„ |j:dRd"gdd… ;¡ }|j<
sâ|j'dddN},t j(| |,|ddO t j )d†|  dQ | ¡ nt j )d‡¡ |j<s0|j'dddN}-t j(| |-|ddO t j )dˆ|  dQ | ¡ nt j )d‰¡ t j dŠ¡ t j d‹¡ t=ƒ  W nL t>k
r¨ }. z,t? @¡ }/t j dŒt|.ƒ d t|/ƒ ¡ W 5 d }.~.X Y nX d S )ŽNzgetting data store infor   r   r   Úcustomer_parametersÚ
dbx_folder©Útzinfoé   é   é   zhttps://api.uphance.com/seasonsÚgetT©ÚurlÚoverrider   z&Uphance Error on Season API call for :é   r)   Úid©ÚinplaceÚ|FÚInt64©Úin_stockÚavailable_to_sellÚavailable_to_sell_from_stockÚ	season_idÚsizeÚean©ÚsepÚ	index_colÚdtypeú https://api.uphance.com/productsz+Request product dump from Uphance - Page : ú/?page=ú&Uphance Product API Call Status Code: ú'Uphance Error on Product API call for :ÚproductsZ
variationsÚskusrR   r   Zp_idZp_identifierZproduct_identifierZp_namer#   rD   Ú	image_urlrP   r"   c                 s   s   | ]}t |ƒV  qd S r   ©r$   )Ú.0Úxr   r   r   Ú	<genexpr>’   s     z&get_data_store_info.<locals>.<genexpr>ÚcategoryÚsub_categoryÚcolorÚsku_idrQ   Ú
sku_numberrM   rN   rO   ZpricesZprice_Z	_currencyÚcurrencyZ_wspZ	wsp_moneyZ_mrspZ
msrp_moneyzStock Data concatÚmetaÚ	next_pagezMemory Analysis in Data Storezgary@mclarenwilliams.com.auc                 S   s   t | d ƒS )NrP   )r,   )Úrowr   r   r   Ú<lambda>©   ó    z%get_data_store_info.<locals>.<lambda>)Úaxisr+   r   Úday_of_weekr   éZ   )ÚcolumnsrI   )rT   Úindex)rE   z$Uphance stock DataStore updated for z
File Path: Úorder_idÚ	order_numÚdate_orderedÚchannelÚqty_orderedÚORÚdate_shippedÚqty_shippedÚqty_varianceÚPCÚ	po_numberÚdate_receivedÚqty_receivedÚboolean)rv   ry   rz   rw   r{   rR   rs   ©rp   ©r~   rR   Úsenté   ©Údays)Ú	from_dateÚ	file_specÚreceivedÚTPr   Úpath_displayzutf=8Ú
ÚHDé   ÚOR1é   ÚAé   ÚOR2é   é   Úclient_modified)ÚsubsetrI   Úignore_indexÚOS1ÚOS2z!Unable to get file from dropbox: z+
Processing of queued files will be abortedÚouter)ÚonÚhowz.0Ú )Úregexc                 S   s    g | ]}d |ksd|kr|‘qS )Ú_xÚ_yr   )r_   Úcr   r   r   Ú
<listcomp>&  s       z'get_data_store_info.<locals>.<listcomp>Úignore)rI   Úerrors)Úas_indexz%Uphance orders DataStore updated for z;Uphance orders DataStore not updated as dataframe was emtpyz.Uphance purchase orders DataStore updated for zDUphance purchase orders DataStore not updated as dataframe was emtpyz finished getting data store infoz&finished updating data from data storez4Error retrieving data from Uphance/nException Info: z
Traceback Info: 
)AÚcommonÚloggerÚdebugÚ
data_storeÚosÚpathr(   Úaccess_secret_versionÚrunning_localr   r   ÚreplaceÚutc_zoneÚ
astimezoneÚto_zoner   r   Úuphance_api_callÚwarningr&   r3   Ú	DataFrameÚ	from_dictÚastyper$   Ú	set_indexÚread_dropbox_bytestreamÚread_csvr!   r:   Ú
send_emailr1   ÚgetvalueÚapplyÚto_datetimeÚdtÚ	dayofweekr…   ÚdropÚto_csvÚstore_dropboxÚinfoÚget_dropbox_file_infor   ÚreadÚdecoder%   Ú
cd_pollingÚget_CD_parameterÚtypeÚrangeÚlenr4   Údrop_duplicatesÚmergerq   rp   ÚtolistÚgroupbyÚfirstÚemptyÚget_data_from_data_storeÚ	ExceptionÚ	tracebackÚ
format_exc)0ÚcustomerÚdata_store_folderÚstock_file_pathÚorders_file_pathÚpo_file_pathZorders_retrieve_pathÚaest_nowZurl_seasonsÚresponseÚdataÚbyte_streamÚdfÚurl_productÚpageZrow_dictÚpr    ÚskuÚprZdf_filteredZcsv_file_dataZstock_columnsZ
po_columnsÚ	orders_dfÚpo_dfÚqueuedFilesZor_dfZpc_dfÚ	file_itemÚ
data_linesÚ	stream_idZ	action_idru   rr   rs   Zeansrv   Úiry   rz   Zpo_idr~   Z	merged_dfZdedup_col_listZorders_csv_file_dataZpo_csv_file_dataÚexÚtbr   r   r   Úget_data_store_infoU   sj   

"

$,,<
($,2


$



 

$

$


rî   c                  C   s"  zÎt jrt ¡ ant ¡ jtd t¡jd dat  	t
t¡} | rdtj| dddddtttdœdant ¡ atjr„t t d¡¡W S t td ¡td< td  ¡  ¡  ¡ atd  ¡  ¡  ¡ att ¡ ƒatt ¡ ƒatt d	d
 a!t  	t
t"¡} | rtj| dddddddtdœda#nt ¡ a#t  	t
t$¡} | rLtj| dddtdœda%nt ¡ a%t%jsdt#jrjW d S t t%d ¡j&jt%d< t t#d ¡j&jt#d< t t#d ¡j&jt#d< td j&jtd< td  't¡td< W nL t(k
r } z,t) *¡ }t j+ ,dt|ƒ d t|ƒ ¡ W 5 d }~X Y nX d S )Nr=   rJ   FrK   rL   rS   z'No Stock Data retrieved from Data Storer   rA   r„   r   )rv   ry   rz   rw   r{   rR   r   r}   rt   rx   rQ   ú/Error Process Dashboard Layout
Exception Info: ú/nTraceback Info: )-r¦   r­   r   r   rÛ   r®   r¯   r°   r±   r¸   rÖ   rØ   r3   r¹   r$   Ústock_info_dfr´   rÑ   r
   ÚDivÚPr½   ÚmaxÚto_pydatetimer   Úlatest_dateÚminÚearliest_dateÚlast_day_of_monthÚdefault_end_season_dateÚget_start_of_previous_weekÚstart_of_previous_weekr   Úend_of_previous_weekrÙ   rå   rÚ   ræ   r¾   r¶   rÓ   rÔ   rÕ   r§   r³   )rÞ   rì   rí   r   r   r   rÒ   B  sD    
""rÒ   c                 C   s&   | j ddtdd }|t|jd S )Né   )Údayr“   r„   )r®   r   rÿ   )Zany_dayZ
next_monthr   r   r   rù   Š  s    rù   c                 C   s   t dd}| | S )Nrƒ   r„   )r   )Z
date_valueÚ
seven_daysr   r   r   rû     s    
rû   c                 C   s(   |d |d | d k|d dk@    ¡ S )Nr   re   rN   r   )r÷   )rj   rß   r   r   r   Úget_earliest_date–  s    r  c                   C   s   t tttttttfS r   )rñ   rå   ræ   rö   rø   rú   rü   rý   r   r   r   r   Úget_data_from_globalsœ  s       þr  c                 C   sB   | ddg | d |k }|j ddiddd |jddd |d S )	NrR   rO   r   Úbase_available_to_sellTrF   )rI   rm   rH   )Úrenamer·   )rß   Úbase_start_dater9   r   r   r   Úget_base_available_to_sell¥  s    r  c                 C   s€   | | d | d @  } t |k r"|}nt }tj t|ƒd ttƒ ¡ | jt | d |k| d tk@ | d d¡d 	d¡ 
d	d
i¡S )Nrw   r{   Z_______rt   rv   r   ©ÚresultrR   r  Úsum)rü   r¦   r§   r¨   r$   rý   ÚassignÚnpÚwhererÏ   Úagg)rß   r  Ú
start_dater   r   r   Úget_last_week_orders­  s    r  c                 C   sF   | | d | d @  } | j t | d |k| d d¡d d¡ dd	i¡S )
Nrw   r{   rt   rv   r   r  rR   r  r	  )r
  r  r  rÏ   r  ©rß   r  r   r   r   Úget_orders_since_start¹  s    r  c                 C   sD   | j t | d |k| d j d¡ @ | d d¡d d¡ dd	i¡S ©
Nr}   r|   ZCRNr~   r   r  rR   r  r	  ©r
  r  r  r$   ÚcontainsrÏ   r  r  r   r   r   Úget_additonal_purchases¾  s    r  c                 C   sB   | j t | d |k| d j d¡@ | d d¡d d¡ dd	i¡S r  r  r  r   r   r   Úget_returnsÂ  s    r  c              
   C   s‚  z.t ƒ \aaaaaaaat	j
 d¡ t ¡ }ttdddg | ƒ d¡}|jddd | |¡}|jdd t	j
 d¡ ||d tk  ¡ }|d	  d
d„ ¡|d< t	j
 t|ddddddg ƒ¡ tt| ƒjddid}|j t¡|_tt| ƒjddid}|j t¡|_tttd dk | ƒjddid}|j t¡|_tttd dk | ƒjddid}|j t¡|_tttd dk | ƒjddid}|j t¡|_tttd dk | ƒjddid}|j t¡|_t	j
 d¡ t	j
 t|jƒ¡ t	j
 t|dddddg ƒ¡ |jddd |j t¡|_| |¡}| |¡}| |¡}| |¡}| |¡}| |¡}|d  d¡|d< |d  d¡|d< |d  d¡|d< |d  d¡|d< |d  d¡|d< |d  d¡|d< |jdd t	j
 d¡ t	j
 t|ƒ¡ |d |d  |d  |d< |d |d  |d < |d |d  |d!< |d |d |d   |d"< |d |d |d   |d#< |d |d  |d  |d$< |d |d  t|  j |d%< |d |d |d   |d&< |d |d%  d' |d(< |d"d#d$d%d(g  tjtj gtj ¡|d"d#d$d%d(g< t	j
 d)¡ t	j
 t|ƒ¡ |W S  t!k
r| }	 z,t" #¡ }
t	j
 $d*t|	ƒ d+ t|
ƒ ¡ W 5 d }	~	X Y nX d S ),NzABegin Manipulation and Merging of Order and PO info into Stock DFrR   r   rO   r  TrH   zCBase data merge complete - starting collection of po and orders DFsrD   c                 S   s   dt | ƒ d S )Nz[![Image Not Available](z)](https://aemery.com)r^   )Úar   r   r   rk   á  rl   zprocess_data.<locals>.<lambda>Zurl_markdownrb   rc   rM   r  Zadditional_purchasesr€   Úreturnsru   zeCommerce - onlineZonline_orders_last_7_dayszeCommerce - reorderZwholesale_orders_last_7_daysZonline_orders_since_startZwholesale_orders_since_startz\Finished collection of po and order info - starting merge of PO and order info into Stock DFr   z7start vectored operations for calculating extra columnsZ
base_stockZprice_eCommerce_mrspZonline_revenue_since_startZwholesale_revenue_since_startZonline_pc_since_startZwholesale_pc_since_startZseasonal_sell_through_pcZdaily_sell_rateZreturn_raterƒ   Zestimated_sell_out_weekszCfinished vectored operations - data manipulation and merge completerï   rð   )%r  rñ   rå   ræ   rö   rø   rú   rü   rý   r¦   r§   r¨   Úcopyr  r  r·   r(   Úreset_indexÚmapr$   r  rq   r¶   r  r  r  rp   Úfillnar…   r®   r  ÚinfÚnanrÓ   rÔ   rÕ   r³   )r  Zbase_stock_info_dfZbase_available_to_sell_dfZadditional_purchases_dfÚ
returns_dfZonline_orders_prev_week_dfZwholesale_orders_prev_week_dfZonline_orders_since_start_dfZwholesale_orders_since_start_dfrì   rí   r   r   r   Úprocess_dataÇ  sx    
 """"





4r   c              
   C   sš   zJt j dtt| ƒƒ d t| ƒ ¡ t| ƒtkrBt | d¡ ¡ } t| ƒW S  t	k
r” } z,t
 ¡ }t j dt|ƒ d t|ƒ ¡ W 5 d }~X Y nX d S )NzBase Start Date in global_storer‹   z%Y-%m-%dz!Exception in Cache Global Store: rð   )r¦   r§   r¨   r$   rÉ   r   Ústrptimer   r   rÓ   rÔ   rÕ   r³   )r  rì   rí   r   r   r   Úglobal_store3  s    $
r"  c                	   C   s    t  ¡  t ¡  W 5 Q R X d S r   )r   Úapp_contextÚcacheÚclearr   r   r   r   Úflush_cache>  s    
r&  c              
   C   s¼  zhd}g }d}i }g |d< d}|rªt j dt|ƒ ¡ t j| d|d t|ƒ dd	}t j d
t|d ƒ ¡ |d rŠt j d|  ¡ qªq |d }| |¡ |d d }q d}|D ]Î}|d D ]À}	|d7 }tj| |	ddd}
|
d  	¡ D ]f\}}|dkrFt j t|ƒ¡ t j t|ƒ¡ |D ]$}t j t|ƒ¡ |d  |¡ qqè|||< qè|dkrh||
d  }q¾|t
 dd|
d ¡ }q¾q²t jr”t ¡ }nt ¡ jtd t¡jd d}d| d¡ d }t j |¡ t | |||¡}t j dt|ƒ ¡ d }|d D ]}d|kr |} qq |rh|d r2d}nd}t jdd|d t|d ƒ d d | dg| d  W nL tk
r¶ } z,t ¡ }t j d!t|ƒ d" t|ƒ ¡ W 5 d }~X Y nX d S )#NrW   r   ÚerrorrF   z>Request product dump from Uphance for master IT dump - Page : rB   rX   FrC   rY   r   rZ   rh   ri   r[   T)ÚmasterrE   zHD\|([A-Z]){2}\|IT
r=   ZIT_FULL_z%Y%m%dz.csvzresult_dict: Ú
send_to_CDzMaster IT file sent to CDzMaster IT file NOT sent to CDzMaster IT file Processingz

Error Info: r‹   zOutput file:
Úglobal)rÖ   z7Exception in retrieving and processing Master IT file: rð   )r¦   r§   r¨   r$   r²   r³   r   Úuphance_webhookÚprocess_product_updateÚitemsÚreÚsubr­   r   r   r®   r¯   r°   r±   ÚstrftimeÚprocess_filerº   rÓ   rÔ   rÕ   )rÖ   rà   Zproduct_pagesÚ	file_dataÚresult_dictrá   rÜ   rÝ   rë   râ   r  r   r    ÚerÛ   Ú	file_nameÚerror_send_to_CDr'  Úerror_messagerì   rí   r   r   r   Úget_master_IT_fileB  sj    







6r8  )Jrª   Úpandasr3   r   r   r   r   Údateutilr   rÔ   Zflask_cachingr   Zmemory_profilerr   r/   Úior	   Únumpyr  Údashr
   r   r   r   r   r.  ÚFlaskApp.app.commonr   r¦   Ú FlaskApp.app.cross_docks_pollingÚcross_docks_pollingrÇ   Ú!FlaskApp.app.uphance_webhook_infoÚuphance_webhook_infor+  ÚFlaskApp.appÚtzutcr¯   Úgettzr±   rÖ   r1   r©   r×   r«   r(   rØ   rÙ   rÚ   r´   rñ   ZCACHE_CONFIGr$  Úinit_appr   r!   r,   r:   rî   rÒ   rù   rû   r  r  r  r  r  r  r  r   Úmemoizer"  r&  r8  r   r   r   r   Ú<module>   sj   

ý


 nH	l

F