Black Lives Matter. Please consider donating to Black Girls Code today.

Realtime streaming of pandas dataframe

Hi,

I am new to Dash and trying to develop a real-time streaming app based on the example: https://dash-gallery.plotly.host/dash-live-model-training

While the original example has a csv which was updating in real-time, I am using a csv file with a python generator to test the app.
My app is working however it is not generating real-time line chart. It can generate only ‘markers’ in real-time. I tried with bar charts as well, they are also working fine. Any particular thing I am missing (I can’t share my work code but can create a dummy file to demonstrate).

Here is my code:

import os

import dash
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import plotly.graph_objs as go
from dash.dependencies import Input, Output, State
from plotly import tools

from demo_utils import demo_components, demo_callbacks, demo_explanation

names = [‘step’, ‘var1’,‘var2’,‘var3’]
LOGFILE = ‘data.csv’

app = dash.Dash(name)
server = app.server

def chunk_generator(filename, header=False, chunk_size=1):
for chunk in pd.read_csv(filename, delimiter=’,’,iterator=True,chunksize=chunk_size,parse_dates=[1]):
yield chunk

def generator1(filename, header=False, chunk_size=1):
chunk = chunk_generator(filename, header=False, chunk_size=1)
for row in chunk:
yield row

run_log_df = generator1(filename=LOGFILE)

Custom Script for Heroku, switch to demo mode when hosted on Heroku

if ‘DYNO’ in os.environ:
app.scripts.append_script({
‘external_url’: ‘https://cdn.rawgit.com/chriddyp/ca0d8f02a1659981a0ea7f013a378bbd/raw/e79f3f789517deec58f41251f7dbb6bee72c44ab/plotly_ga.js
})
demo_mode = True
else:
demo_mode = False

def div_graph(name):
“”“Generates an html Div containing graph and control options for smoothing and display, given the name”""
return html.Div([
html.Div(
id=f’div-{name}-graph’,
className=“ten columns”
),

    html.Div([
        html.Div([
            html.P(f'{name}: ', style={'font-weight': 'bold', 'margin-bottom': '0px'}),
        ],
            style={'margin-top': '10px'}
        ),

        html.Div([
            html.P("Plot Display mode:", style={'font-weight': 'bold', 'margin-bottom': '0px'}),

            dcc.RadioItems(
                options=[
                    {'label': ' Overlapping', 'value': 'overlap'},
                    {'label': ' Separate (Vertical)', 'value': 'separate_vertical'},
                    {'label': ' Separate (Horizontal)', 'value': 'separate_horizontal'}
                ],
                value='overlap',
                id=f'radio-display-mode-{name}'
            ),

            html.Div(id=f'div-current-{name}-value')
        ]),
    ],
        className="two columns"
    ),
],
    className="row"
)

traces=[]

app.layout = html.Div([
# Banner display
html.Div([
html.H2(
‘Live Model Training Viewer’,
id=‘title’
),
html.Img(
src=“https://s3-us-west-1.amazonaws.com/plotly-tutorials/logo/new-branding/dash-logo-by-plotly-stripe-inverted.png
)
],
className=“banner”
),

# Body
html.Div([
    html.Div([
        dcc.Dropdown(
            id='dropdown-interval-control',
            options=[
                {'label': 'No Updates', 'value': 'no'},
                {'label': 'Slow Updates', 'value': 'slow'},
                {'label': 'Regular Updates', 'value': 'regular'},
                {'label': 'Fast Updates', 'value': 'fast'}
            ],
            value='regular',
            className='ten columns',
            clearable=False,
            searchable=False
        ),

        html.Div(
            id="div-step-display",
            className="two columns"
        )
    ],
        id='div-interval-control',
        className='row'
    ),

    dcc.Interval(
        id="interval-log-update",
        interval=1*1000,
        n_intervals=0
    ),

    # Hidden Div Storing JSON-serialized dataframe of run log
    html.Div(id='run-log-storage', style={'display': 'none'}),

    # The html divs storing the graphs and display parameters
    div_graph('accuracy'),
    #div_graph('cross-entropy'),

    # Explanation for the demo version of the app
    #demo_explanation(demo_mode)
],
    className="container"
)

])

def update_graph(graph_id,
graph_title,
run_log_json,
display_mode,
yaxis_title):
“”"
:param graph_id: ID for Dash callbacks
:param graph_title: Displayed on layout
:param y_train_index: name of column index for y train we want to retrieve
:param y_val_index: name of column index for y val we want to retrieve
:param run_log_json: the json file containing the data
:param display_mode: ‘separate’ or ‘overlap’
:param checklist_smoothing_options: ‘train’ or ‘val’
:param slider_smoothing: value between 0 and 1, at interval of 0.05
:return: dcc Graph object containing the updated figures
“”"

if run_log_json:  # exists
    layout = go.Layout(title=graph_title,showlegend=False,margin=go.Margin(l=50, r=50, b=50, t=50),yaxis={'title': yaxis_title})

    run_log_df = pd.read_json(run_log_json, orient='split')

    step = run_log_df['step']
    y_train = run_log_df['var1']
    y_val = run_log_df['var2']

    trace_train = go.Scatter(
        x=step,
        y=y_train,
        mode='lines+markers',
        name='Training'
    )

    traces.append(trace_train)

    trace_val = go.Scatter(
        x=step,
        y=y_val,
        mode='lines+markers',
        name='Validation'
    )

    traces.append(trace_val)

    if display_mode == 'separate_vertical':
        figure = tools.make_subplots(rows=2,
                                     cols=1,
                                     print_grid=False,
                                     shared_yaxes=True)

        figure.append_trace(trace_train, 1, 1)
        figure.append_trace(trace_val, 2, 1)

        figure['layout'].update(title=layout.title,
                                margin=layout.margin,
                                scene={'domain': {'x': (0., 0.5), 'y': (0.5,1)}})

    elif display_mode == 'separate_horizontal':
        figure = tools.make_subplots(rows=1,
                                     cols=2,
                                     shared_yaxes=True,
                                     print_grid=False)

        figure.append_trace(trace_train, 1, 1)
        figure.append_trace(trace_val, 1, 2)

        figure['layout'].update(title=layout.title,
                                margin=layout.margin)

    elif display_mode == 'overlap':
        figure = go.Figure(
            data=traces,
            layout=layout
        )

    else:
        figure = None

    return dcc.Graph(figure=figure, id=graph_id)

return dcc.Graph(id=graph_id)

@app.callback(Output(‘interval-log-update’, ‘interval’),
[Input(‘dropdown-interval-control’, ‘value’)])
def update_interval_log_update(interval_rate):
if interval_rate == ‘fast’:
return 500

elif interval_rate == 'regular':
    return 1000

elif interval_rate == 'slow':
    return 5 * 1000

# Refreshes every 24 hours
elif interval_rate == 'no':
    return 24 * 60 * 60 * 1000

if not demo_mode:
@app.callback(Output(‘run-log-storage’, ‘children’),
[Input(‘interval-log-update’, ‘n_intervals’)])
def get_run_log(_):
names = [‘step’, ‘var1’,‘var2’,‘var3’]

    try:
        temp = next(run_log_df).values.tolist()
        temp_df = pd.DataFrame(temp, columns=names)
        json = temp_df.to_json(orient='split')
    except FileNotFoundError as error:
        print(error)
        print("Please verify if the csv file generated by your model is placed in the correct directory.")
        return None

    return json

@app.callback(Output(‘div-step-display’, ‘children’),
[Input(‘run-log-storage’, ‘children’)])
def update_div_step_display(run_log_json):
if run_log_json:
run_log_df = pd.read_json(run_log_json, orient=‘split’)
return html.H6(f"Step: {run_log_df[‘step’].iloc[-1]}", style={‘margin-top’: ‘3px’})

@app.callback(Output(‘div-accuracy-graph’, ‘children’),
[Input(‘run-log-storage’, ‘children’),
Input(‘radio-display-mode-accuracy’, ‘value’)])
def update_accuracy_graph(run_log_json,
display_mode):
graph = update_graph(‘accuracy-graph’,
‘Prediction Accuracy’,
run_log_json,
display_mode,
‘Accuracy’)

# try:
#     if display_mode in ['separate_horizontal', 'overlap']:
#         graph.figure.layout.yaxis['range'] = [0, 1]
#     else:
#         graph.figure.layout.yaxis1['range'] = [0, 1]
#         graph.figure.layout.yaxis2['range'] = [0, 1]

# except AttributeError:
#     pass

return [graph]

@app.callback(Output(‘div-current-accuracy-value’, ‘children’),
[Input(‘run-log-storage’, ‘children’)])
def update_div_current_accuracy_value(run_log_json):
if run_log_json:
run_log_df = pd.read_json(run_log_json, orient=‘split’)
return [
html.P(
“Current Accuracy:”,
style={
‘font-weight’: ‘bold’,
‘margin-top’: ‘15px’,
‘margin-bottom’: ‘0px’
}
),
html.Div(f"Training: {run_log_df[‘var1’].iloc[-1]:.4f}"),
html.Div(f"Validation: {run_log_df[‘var2’].iloc[-1]:.4f}")
]

external_css = [
https://cdnjs.cloudflare.com/ajax/libs/normalize/7.0.0/normalize.min.css”, # Normalize the CSS
https://fonts.googleapis.com/css?family=Open+Sans|Roboto” # Fonts
https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css”,
https://cdn.rawgit.com/xhlulu/0acba79000a3fd1e6f552ed82edb8a64/raw/dash_template.css”,
https://rawgit.com/plotly/dash-live-model-training/master/custom_styles.css
]

for css in external_css:
app.css.append_css({“external_url”: css})

Running the server

if name == ‘main’:
app.run_server(debug=True)

Can anybody help me with a cue what am I doing wrong here?