monitor_jo.py

#!/usr/bin/env python
# coding: utf-8

# # SMNA-Dashboard
# 
# Este notebook trata da apresentação dos resultados do GSI em relação à minimização da função custo do 3DVar. A apresentação dos resultados é feita a partir da leitura de um arquivo CSV e os gráficos são mostrados em um dashboard do Panel para explorar as informações nele contidas. Para mais informações sobre o arquivo CSV e a sua estrutura de dados, veja o notebook `SMNA-Dashboard-load_files_create_dataframe_save.ipynb`.
# 
# Para realizar o deploy do dashboard no GitHub, é necessário converter este notebook em um script executável, o que pode ser feito a partir da interface do Jupyter (File -> Save and Export Notebook As... -> Executable Script). A seguir, utilize o comando abaixo para converter o script em uma página HTML. Junto com a página, será gerado um arquivo JavaScript e ambos devem ser adicionados ao repositório, junto com o arquivo CSV.
# 
# ```
# panel convert SMNA-Dashboard.py --to pyodide-worker --out .
# ```
# 
# Para utilizar o dashboard localmente, utilize o comando a seguir:
# 
# ```
# panel serve SMNA-Dashboard.ipynb --autoreload --show
# ```
# 
# ---
# Carlos Frederico Bastarz (carlos.bastarz@inpe.br), Abril de 2023.

import io
import os
import re
import numpy as np
import pandas as pd
import hvplot.pandas
import panel as pn
#from panel_modal import Modal
from datetime import datetime, timedelta
from matplotlib import pyplot as plt

from monitor_dates import MonitoringAppDates

from monitor_texts import MonitoringAppTexts

monitor_app_texts = MonitoringAppTexts()
monitor_warning_bottom_main = monitor_app_texts.warnings()

pn.extension('floatpanel')
pn.extension(sizing_mode='stretch_width', notifications=True)

#dfs = pd.read_csv('https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/jo/jo_table_series.csv', header=[0, 1], parse_dates=[('df_preOper', 'Date'), ('df_preOper_new', 'Date'), ('df_JGerd', 'Date')])
#dfs = pd.read_csv('https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/jo/jo_table_series.csv', header=[0, 1], parse_dates=[('df_preOper', 'Date'), ('df_JGerd', 'Date')])
dfs = pd.read_csv('https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/jo/jo_table_series.csv', header=[0, 1], parse_dates=[('df_preOper', 'Date')])

# Separa os dataframes de interesse
df_preOper = dfs.df_preOper
#df_preOper_new = dfs.df_preOper_new
#df_JGerd = dfs.df_JGerd

# Atribui nomes aos dataframes
df_preOper.name = 'df_preOper'
#df_preOper_new.name = 'df_preOper_new'
#df_JGerd.name = 'df_JGerd'

monitoring_app_dates = MonitoringAppDates()
sdate = monitoring_app_dates.getDates()[0].strip()
edate = monitoring_app_dates.getDates()[1].strip()

start_date = datetime(int(sdate[0:4]), int(sdate[4:6]), int(sdate[6:8]), int(sdate[8:10]))
start_date_fixed = datetime(int(sdate[0:4]), int(sdate[4:6]), int(sdate[6:8]), int(sdate[8:10]))
end_date = datetime(int(edate[0:4]), int(edate[4:6]), int(edate[6:8]), int(edate[8:10]))
end_date_fixed = datetime(int(edate[0:4]), int(edate[4:6]), int(edate[6:8]), int(edate[8:10]))

# Constrói as widgets e apresenta o dashboard
#sdate = pd.to_datetime(dfs.df_preOper['Date']).dt.strftime('%Y%m%d%H').min()
#edate = pd.to_datetime(dfs.df_preOper['Date']).dt.strftime('%Y%m%d%H').max()
#sdate = pd.to_datetime(sfdate.min()).date()
#edate = pd.to_datetime(efdate.max()).date()
#start_date = sdate.datetime.strptime('%Y%m%d%H')
#end_date = edate.datetime.strptime('%Y%m%d%H')
#start_date = datetime(2024, 3, 12, 12)
#start_date_fixed = datetime(2024, 3, 19, 12)
#end_date = datetime(2024, 3, 12, 12)
#end_date_fixed = datetime(2024, 3, 19, 12)

values = (start_date, end_date)

date_range_slider = pn.widgets.DatetimeRangePicker(name='Date Range', value=values, enable_time=False, width=240)

#experiment_list = [df_preOper, df_preOper_new, df_JGerd]
#experiment_list2 = ['df_preOper', 'df_preOper_new', 'df_JGerd']
#experiment_list = [df_preOper, df_JGerd]
#experiment_list2 = ['df_preOper', 'df_JGerd']
experiment_list = [df_preOper]
experiment_list2 = ['df_preOper']
variable_list = ['surface pressure', 'temperature', 'wind', 'moisture', 'gps', 'radiance'] 
synoptic_time_list = ['00Z', '06Z', '12Z', '18Z', '00Z e 12Z', '06Z e 18Z', '00Z, 06Z, 12Z e 18Z']
iter_fcost_list = ['OMF', 'OMF (1st INNER LOOP)', 'OMF (2nd INNER LOOP)', 'OMA (AFTER 1st OUTER LOOP)', 'OMA (1st INNER LOOP)', 'OMA (2nd INNER LOOP)', 'OMA (AFTER 2nd OUTER LOOP)']

date_range = date_range_slider.value

experiment = pn.widgets.MultiChoice(name='Experiments (Plots)', value=[experiment_list[0].name], options=[i.name for i in experiment_list], solid=False, width=240)
experiment2 = pn.widgets.Select(name='Experiment (Table)', value=experiment_list[0].name, options=[i.name for i in experiment_list], disabled=True, width=240)
variable = pn.widgets.Select(name='Variable', value=variable_list[0], options=variable_list, width=240)
synoptic_time = pn.widgets.RadioBoxGroup(name='Synoptic Time', value=synoptic_time_list[-1], options=synoptic_time_list, inline=False, width=240)
iter_fcost = pn.widgets.Select(name='Iteration', value=iter_fcost_list[0], options=iter_fcost_list, width=240)

# Considerando que todos os dataframes possuem o mesmo tamanho (i.e, linhas e colunas), 
# então a função a seguir utiliza apenas um dos dataframes para criar a máscara temporal que será 
# utilizada pelos demais
def subset_dataframe(df, start_date, end_date, send_notification):
    if start_date > end_date_fixed:
        start_date = start_date_fixed
        if send_notification:
            pn.state.notifications.error('Seleção da data inicial é maior do que a data final no arquivo CVS', duration=5000)
    if end_date > end_date_fixed:
        end_date = end_date_fixed
        if send_notification:
            pn.state.notifications.error('Seleção da data final é maior do que a data final no arquivo CVS', duration=5000)
    mask = (df['Date'] >= start_date) & (df['Date'] <= end_date)
    return df.loc[mask]

height=250

@pn.depends(variable, experiment, synoptic_time, iter_fcost, date_range_slider.param.value)
def plotCurves(variable, experiment, synoptic_time, iter_fcost, date_range):
    for count, i in enumerate(experiment):
        if count == 0:
            sdf = globals()[i]
            df = dfs.xs(sdf.name, axis=1)
            
            send_notification = True
            start_date, end_date = date_range
            df2 = subset_dataframe(df, start_date, end_date, send_notification)
            
            if synoptic_time == '00Z': time_fmt0 = '00:00:00'; time_fmt1 = '00:00:00'
            if synoptic_time == '06Z': time_fmt0 = '06:00:00'; time_fmt1 = '06:00:00'
            if synoptic_time == '12Z': time_fmt0 = '12:00:00'; time_fmt1 = '12:00:00'
            if synoptic_time == '18Z': time_fmt0 = '18:00:00'; time_fmt1 = '18:00:00'    
    
            if synoptic_time == '00Z e 12Z': time_fmt0 = '00:00:00'; time_fmt1 = '12:00:00'
            if synoptic_time == '06Z e 18Z': time_fmt0 = '06:00:00'; time_fmt1 = '18:00:00'
    
            if synoptic_time == '00Z e 06Z': time_fmt0 = '00:00:00'; time_fmt1 = '06:00:00'
            if synoptic_time == '12Z e 18Z': time_fmt0 = '12:00:00'; time_fmt1 = '18:00:00'    
    
            if synoptic_time == '00Z, 06Z, 12Z e 18Z': time_fmt0 = '00:00:00'; time_fmt1 = '18:00:00'    
    
            if time_fmt0 == time_fmt1:
                df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').at_time(str(time_fmt0)).reset_index()
            else:                
                df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')
                
                if synoptic_time == '00Z e 12Z':
                    df_s = df_s.drop(df_s.at_time('06:00:00').index).reset_index()
                elif synoptic_time == '06Z e 18Z':    
                    df_s = df_s.drop(df_s.at_time('12:00:00').index).reset_index()
                elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                    df_s = df_s.reset_index()                    
                
            xticks = len(df_s['Date'].values)    
                
            ax_nobs = df_s.hvplot.line(x='Date', y='Nobs', xlabel='Date', ylabel=str('Nobs'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)    
            ax_jo = df_s.hvplot.line(x='Date', y='Jo', xlabel='Date', ylabel=str('Jo'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)    
            ax_jon = df_s.hvplot.line(x='Date', y='Jo/n', xlabel='Date', ylabel=str('Jo/n'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)
            
            # Adiciona pontos às curvas
            sax_nobs = df_s.hvplot.scatter(x='Date', y='Nobs', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')    
            sax_jo = df_s.hvplot.scatter(x='Date', y='Jo', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')     
            sax_jon = df_s.hvplot.scatter(x='Date', y='Jo/n', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')             
            
        else:
            
            sdf = globals()[i]
            df = dfs.xs(sdf.name, axis=1)
            
            send_notification = False
            start_date, end_date = date_range
            df2 = subset_dataframe(df, start_date, end_date, send_notification)
            
            if synoptic_time == '00Z': time_fmt0 = '00:00:00'; time_fmt1 = '00:00:00'
            if synoptic_time == '06Z': time_fmt0 = '06:00:00'; time_fmt1 = '06:00:00'
            if synoptic_time == '12Z': time_fmt0 = '12:00:00'; time_fmt1 = '12:00:00'
            if synoptic_time == '18Z': time_fmt0 = '18:00:00'; time_fmt1 = '18:00:00'    
    
            if synoptic_time == '00Z e 12Z': time_fmt0 = '00:00:00'; time_fmt1 = '12:00:00'
            if synoptic_time == '06Z e 18Z': time_fmt0 = '06:00:00'; time_fmt1 = '18:00:00'
    
            if synoptic_time == '00Z, 06Z, 12Z e 18Z': time_fmt0 = '00:00:00'; time_fmt1 = '18:00:00'   
    
            if time_fmt0 == time_fmt1:
                df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').at_time(str(time_fmt0)).reset_index()
            else:                    
                df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

                if synoptic_time == '00Z e 12Z':
                    df_s = df_s.drop(df_s.at_time('06:00:00').index).reset_index()
                elif synoptic_time == '06Z e 18Z':    
                    df_s = df_s.drop(df_s.at_time('12:00:00').index).reset_index()
                elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                    df_s = df_s.reset_index()
                
            xticks = len(df_s['Date'].values)
            
            ax_nobs *= df_s.hvplot.line(x='Date', y='Nobs', xlabel='Date', ylabel=str('Nobs'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)
            ax_jo *= df_s.hvplot.line(x='Date', y='Jo', xlabel='Date', ylabel=str('Jo'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)
            ax_jon *= df_s.hvplot.line(x='Date', y='Jo/n', xlabel='Date', ylabel=str('Jo/n'), persist=True, rot=90, grid=True, label=str(i), line_width=3, height=height, responsive=True)
            
            # Adiciona pontos às curvas
            sax_nobs *= df_s.hvplot.scatter(x='Date', y='Nobs', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')    
            sax_jo *= df_s.hvplot.scatter(x='Date', y='Jo', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')     
            sax_jon *= df_s.hvplot.scatter(x='Date', y='Jo/n', height=height, label=str(i), persist=True, responsive=True).opts(size=5, marker='o')             
            
    return pn.Column(ax_nobs*sax_nobs, ax_jo*sax_jo, ax_jon*sax_jon, sizing_mode='stretch_width')

@pn.depends(variable, experiment2, synoptic_time, iter_fcost, date_range_slider.param.value)
def getTable(variable, experiment2, synoptic_time, iter_fcost, date_range):
    #for count, i in enumerate(experiment):
    #    if count == 0:
    sdf = globals()[experiment2]
    df = dfs.xs(sdf.name, axis=1)
    
    send_notification = False            
    start_date, end_date = date_range
    df2 = subset_dataframe(df, start_date, end_date, send_notification)
            
    if synoptic_time == '00Z': time_fmt0 = '00:00:00'; time_fmt1 = '00:00:00'
    if synoptic_time == '06Z': time_fmt0 = '06:00:00'; time_fmt1 = '06:00:00'
    if synoptic_time == '12Z': time_fmt0 = '12:00:00'; time_fmt1 = '12:00:00'
    if synoptic_time == '18Z': time_fmt0 = '18:00:00'; time_fmt1 = '18:00:00'    
    
    if synoptic_time == '00Z e 12Z': time_fmt0 = '00:00:00'; time_fmt1 = '12:00:00'
    if synoptic_time == '06Z e 18Z': time_fmt0 = '06:00:00'; time_fmt1 = '18:00:00'
    
    if synoptic_time == '00Z e 06Z': time_fmt0 = '00:00:00'; time_fmt1 = '06:00:00'
    if synoptic_time == '12Z e 18Z': time_fmt0 = '12:00:00'; time_fmt1 = '18:00:00'    
    
    if synoptic_time == '00Z, 06Z, 12Z e 18Z': time_fmt0 = '00:00:00'; time_fmt1 = '18:00:00'    
    
    if time_fmt0 == time_fmt1:
        df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').at_time(str(time_fmt0)).reset_index()
    else:                
        df_s = df2.loc[df2['Observation Type'] == variable].loc[df2['Iter'] == iter_fcost].set_index('Date').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')
                
    if synoptic_time == '00Z e 12Z':
        df_s = df_s.drop(df_s.at_time('06:00:00').index).reset_index()
    elif synoptic_time == '06Z e 18Z':    
        df_s = df_s.drop(df_s.at_time('12:00:00').index).reset_index()
    elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
        df_s = df_s.reset_index()                    
                
    stylesheet = """
        .tabulator-cell {
        font-size: 12px;
    }
    """

    #return pn.Column(df_s, sizing_mode='stretch_width')
    gt_table = pn.widgets.Tabulator(df_s, 
                                    #selectable=False, 
                                    show_index=False,
                                    theme='bootstrap4',
                                    text_align='center',
                                    #layout='fit_data_table',
                                    widths=250,
                                    selectable='toggle',
                                    #layout='fit_data_fill', 
                                    #width=250,
                                    stylesheets=[stylesheet],
                                    disabled=True)
    
    def get_csv():
        io_buffer = io.BytesIO()
        df_s.to_csv(io_buffer, index=False)
        io_buffer.seek(0)  # Retorna ao início do buffer
        return io_buffer

    file_download = pn.widgets.FileDownload(
        icon='download',
        callback=get_csv, 
        filename='jo_table.csv',
        button_type='success',
        width=310
        )

    return pn.Column(gt_table, file_download, sizing_mode='stretch_width')

###

# def Layout():

#     text_info = """
#     # SMNA Dashboard - Função Custo
    
#     ## Curvas
    
#     A depender da quantidade de outer e inner loops, o GSI registra um número diferente de informações sobre o número de observações consideradas (`Nobs`), o custo da minimização (`Jo`) e o custo da minimização normalizado pelo número de observações (`Jo/n`). A configuração do GSI/3DVar aplicado ao SMNA (válido para a data de escrita deste notebook), considera `miter=2` e `niter=3`, ou seja, 2 outer loops com 3 inner loops cada. Nesse sentido, as informações obtidas a partir das iterações do processo de minimização da função custo, consideram o seguinte:
    
#     * `OMF`: início do primeiro outer loop, onde o estado do sistema é dado pelo background;
#     * `OMF (1st INNER LOOP)`: final do primeiro inner loop do primeiro outer loop, onde o estado do sistema ainda é dado pelo background;
#     * `OMF (2nd INNER LOOP)`: final do segundo inner loop do primeiro outer loop, onde o estado do sistema ainda é dado pelo background;
#     * `OMA (AFTER 1st OUTER LOOP)`: início do segundo outer loop, onde o estado do sistema é dado pela análise;
#     * `OMA (1st INNER LOOP)`: final do primeiro inner loop do segundo outer loop, onde o estado do sistema é dado pela análise;
#     * `OMA (2nd INNER LOOP)`: final do segundo inner loop do segundo outer loop, onde o estado do sistema é dado pela análise;
#     * `OMA (AFTER 2nd OUTER LOOP)`: final do segundo outer loop, análise final.
    
#     **Nota:** as informações das iterações `OMF` e `OMF (1st INNER LOOP)` são iguais, assim como as informações das iterações `OMA (AFTER 1st OUTER LOOP)` e `OMA (1st INNER LOOP)`.
    
#     ## Experimentos
    
#     * `df_dtc`: experimento controle SMNA-Oper, com a matriz **B** do DTC, realizado pelo DIMNT;
#     * `df_dtc_alex`: experimento SMNA-Oper, com a matriz **B** do DTC, realizado pela DIPTC;
#     * `df_bamh_T0`: experimento controle SMNA-Oper, com a matriz **B** do BAMH (exp. T0), realizado pelo DIMNT;
#     * `df_bamh_T4`: experimento controle SMNA-Oper, com a matriz **B** do BAMH (exp. T4), realizado pelo DIMNT;
#     * `df_bamh_GT4AT2`: experimento controle SMNA-Oper, com a matriz **B** do BAMH (exp. GT4AT2), realizado pelo DIMNT;
    
#     **Nota:** a descrição dos experimentos T0, T4 e GT4AT2 podem ser encontradas em [https://projetos.cptec.inpe.br/issues/11766](https://projetos.cptec.inpe.br/issues/11766).        
    
#     ## Período
    
#     O período considerado para a apresentação dos resultados é 2023021600 a 2023031600.
    
#     ---
    
#     Atualizado em: 09/05/2023 ([carlos.bastarz@inpe.br](mailto:carlos.bastarz@inpe.br))
    
#     """
  
#     card_parameters = pn.Card(pn.Row(date_range_slider, pn.widgets.TooltipIcon(value='Choose a date range', align='start')), 
#                               pn.Row(variable, pn.widgets.TooltipIcon(value='Choose a variable', align='start')),
#                               pn.Row(iter_fcost, pn.widgets.TooltipIcon(value='Choose the cost function iteration', align='start')), 
#                               pn.Row(synoptic_time, pn.widgets.TooltipIcon(value='Choose one or more synoptic times', align='start')), 
#                               pn.Row(experiment2, pn.widgets.TooltipIcon(value='Choose one experiment', align='start')), 
#                               pn.Column(pn.Row(experiment, pn.widgets.TooltipIcon(value='Choose one or more experiments', align='start')), height=240),  
#                               title='Parameters', collapsed=False)
    
#     settings = pn.Column(card_parameters)

#     tabs_contents = pn.Tabs(('PLOTS', plotCurves), ('TABLE', getTable), active=0)

#     return pn.Column(settings, tabs_contents)

def monitor_jo_sidebar():
    card_parameters = pn.Card(pn.Row(date_range_slider, pn.widgets.TooltipIcon(value='Choose a date range', align='start')), 
                              pn.Row(variable, pn.widgets.TooltipIcon(value='Choose a variable', align='start')),
                              pn.Row(iter_fcost, pn.widgets.TooltipIcon(value='Choose the cost function iteration', align='start')), 
                              pn.Row(synoptic_time, pn.widgets.TooltipIcon(value='Choose one or more synoptic times', align='start')), 
                              pn.Row(experiment2, pn.widgets.TooltipIcon(value='Choose one experiment', align='start')), 
                              pn.Column(pn.Row(experiment, pn.widgets.TooltipIcon(value='Choose one or more experiments', align='start')), height=240), 
                              title='Parameters', collapsed=False)    




    settings = pn.Column(card_parameters)
    return settings

def monitor_jo_main():
    #tabs_contents_jb = pn.Tabs(('PLOTS', plotCurves), ('TABLE', getTable), active=0)
    tabs_contents_jo = pn.Tabs(('PLOTS', plotCurves), ('TABLE', getTable), dynamic=True, active=0)
    tabs_contents = pn.Tabs(('Jo', pn.Column('Jo minimization diagnostics.', tabs_contents_jo)), dynamic=True)#, 
                            #('Jb', pn.Column('Jb minimization diagnostics.', tabs_contents_jb)),
                            #('Jc', pn.Column('Jc minimization diagnostics.', tabs_contents_jo)), active=1)
    main_text = pn.Column("""
    # Minimization Plots

    Navigate through the tabs to visualize the minimization results. Set the parameters on the left to update the curves. Click on the `TABLE` tab to visualize the tabular data.
    """)

    # Callback to enable/disable the experiment2 widget based on the active tab
    def update_experiment2_widget(event):
        if event.new == 1:  # Tab index for "TABLE"
            experiment2.disabled = False
        else:
            experiment2.disabled = True

    # Attach the callback to the active parameter of the tabs
    tabs_contents_jo.param.watch(update_experiment2_widget, 'active')

    return pn.Column(main_text, tabs_contents, monitor_warning_bottom_main, sizing_mode='stretch_width')