Hello!
I’m working on building a dash app for analyzing long-term spacecraft telemetry and am coming across an issue where the figure traces are not updating upon zoom.
Due to there being a large volume of data (several million rows) and the need to plot many metrics against each other, for performance purposes, I’m using lttb to downsample the data to 5000 points if the length of the pandas dataframe is greater than 5000.
When I zoom in, I want the original dataframe to be filtered down to the rows between the selected timestamps (that part is successful), downsampled to 5000 points if its length is greater than 5000 (that part also works), and then the appropriate trace be updated with the new data. That last part appears to work, but the rendered figure still only shows the data as it was originally downsampled when the data was originally loaded.
I’m storing the figure in the variable fig_holder for later retrieval/updating and the original dataframe(s) in dataframe_holder. Below are my callbacks.
Callback to Upload and Generate/Update Figure Upon Data Upload
@app.callback(Output('telemetry-graph-holder', 'children'),
Output('telemetry-output-files', 'children'),
Input('upload-data', 'contents'),
Input('upload-data', 'filename'),
State('telemetry-output-files', 'children'),
State('telemetry-graph-holder', 'children')
)
def update_graph(list_of_contents, list_of_filenames, list_of_files, list_of_figures):
#Check if csv contents is not None
if list_of_contents is not None:
#Check if a figure exists
if list_of_figures is not None:
#Set fig = existing figure
fig = go.Figure(list_of_figures[0]['props']['figure'])
#Instantiate new figure
else: fig = go.Figure()
#Define div for telemetry info in sidebar
TELEM_DIV_STYLE = {
"margin-left": "1rem",
"margin-right": "1rem",
'margin-top': '1rem',
'lineHeight': '60px',
'borderWidth': '1px',
'borderStyle': 'solid',
'borderRadius': '5px',
}
#Iterate through upload content and generate/update figure
for content, filename in zip(list_of_contents, list_of_filenames):
content_type, content_string = content.split(',')
decoded = base64.b64decode(content_string)
#Get name of telemetry
telem_name = filename[:-4]
try:
if 'csv' in filename:
# Assume that the user uploaded a CSV file
df = pd.read_csv(
io.StringIO(decoded.decode('utf-8')))
elif 'xls' in filename:
# Assume that the user uploaded an excel file
df = pd.read_excel(io.BytesIO(decoded))
except Exception as e:
print(e)
return html.Div([
'There was an error processing this file.'
])
#Keep only relevant columns
columns_to_keep = ['timestamp', telem_name]
#Trim df to only wanted columns
df = df[columns_to_keep]
#Filter dataframe to times of interest
df = df[(df['timestamp'] >= START_TIME) & (df['timestamp'] <= END_TIME)]
#Convert timestamps to ints
df['timestamp'] = df['timestamp'].astype(int)
#Remove duplicate rows
duplicate_mask = df['timestamp'].duplicated()
df = df[~duplicate_mask]
#Create a Date column with human-readable values
df['Date'] = pd.to_datetime(df['timestamp'], unit='s')
#Sort dataframe by timestamps
df.sort_values(by="timestamp", inplace=True)
#Append df to df_holder for later access
df_holder.append(df)
#Downdambple data
downsampled_data = lttb.downsample(np.array(df[['timestamp', telem_name]]), n_out=downsample_to)
downsampled_data = downsampled_data.T
#Get downsampled df['Date'] for x-axis
downsampled_dates = df['Date'][df['timestamp'].isin(downsampled_data[0].astype(int))]
#Add trace to figure with new data
fig.add_trace(
go.Scattergl(
x = downsampled_dates,
y = downsampled_data[1],
name = telem_name,
meta = [telem_name for i in range(len(downsampled_data[0]))],
text = [d for d in downsampled_dates],
hovertemplate =
'<b>%{meta}</b><br>' +
'<b>Value:</b> %{y}<br>' +
'<b>Timestamp:</b> %{text}<extra></extra>',
mode = 'markers+lines',
marker = dict(
size = 3
),
line = dict(
width=1
),
showlegend = True
)
)
#Update layout
fig.update_layout(
legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="right",
x=1
)
)
#Define telemetry div for telem info in sidebar
telem_div = html.Div(
id=f'{telem_name}_div',
children = [
html.B(telem_name,
style = {
'font-size': '12px'
})
],
style=TELEM_DIV_STYLE
)
#Append telemetry info to list already in sidebar
if list_of_files is not None:
list_of_files.append(telem_div)
else: list_of_files = [telem_div]
#Create graph object to to display
graph_obj = dcc.Graph(
id='telemetry-graph',
figure=fig
)
#Store fig in fig_holder for later reference
if fig_holder:
fig_holder[0] = fig
else: fig_holder.append(fig)
#Return objects
return [graph_obj], list_of_files
The above callback works as expected and there is no issue there.
Callback to Update Trace(s) on Zoom
@app.callback(Output('telemetry-graph', 'figure'),
[Input('telemetry-graph', 'relayoutData')])
def relayout_graph(relayout_data):
if relayout_data is not None and 'xaxis.range[0]' in relayout_data:
#Get lower and upper x_range values
x_range = [relayout_data['xaxis.range[0]'], relayout_data['xaxis.range[1]']]
#retrieve fig from fig_holder[0]
fig = fig_holder[0]
#Iterate through dataframes in df_holder
for df in df_holder:
print(df.info())
#Filter dataframe to times of interest
df = df[(df['Date'] >= x_range[0]) & (df['Date'] <= x_range[-1])]
print(df.info())
#Get name of telemetry to extract from df
not_telemetry_columns = ['timestamp', 'Date']
telem_name = [x for x in list(df.columns) if x not in not_telemetry_columns][0]
#If len(df) is greater than some value (5000 currently) downsample df
if len(df) > downsample_to:
#Downdambple data
downsampled_data = lttb.downsample(np.array(df[['timestamp', telem_name]]), n_out=downsample_to)
downsampled_data = downsampled_data.T
#Print length of downsampled data - used for rough debugging atm
print(len(downsampled_data[0]))
#Get downsampled df['Date'] for x-axis
downsampled_dates = df['Date'][df['timestamp'].isin(downsampled_data[0].astype(int))]
#Update appropriate trace
fig.update_traces(
x = downsampled_dates,
y = downsampled_data[1],
selector = dict(
name = telem_name
)
)
#If len(df) is less than some value (5000 in this case) don't downsample
else:
#Update appropriate trace
fig.update_traces(
x = df['Date'],
y = df[telem_name],
selector = dict(
name = telem_name
)
)
#Print length of x and y data in fig - used for rough debugging with previous print
print(len(fig.data[0]['x']), len(fig.data[0]['y']))
#Store updated fig in fig_holder for later retrieval/update
fig_holder[0] = fig
#Return fig to 'telemetry-graph': figure
return fig
return
This callback appears to correctly update the fig trace x and y values, but the rendered fig does not reflect that i.e I would expect there to always be 5000 datapoints in a trace unless the length of the dataframe is less than 5000, but that is not the case.
The below images reflect the figure upon initial generation (first image), and after zooming in considerably (second image). In the second image, the length of the x and y properties is 2089, but there are not that many datapoints rendered.
Inital Figure on Data Upload
Figure After Zooming
Any help would be greatly appreciated.
Thank you!