Hi,
I am trying to create heatmap to show the status of sensor nodes.
The data is somewhat like
State | filename | vsn | Count_Parameters | Percentage_of_sensor_reading_data |
---|---|---|---|---|
Up | 05.30 | 0EA | 27 | 50 |
Up | 05.45 | 0EA | 27 | 50 |
Up | 06.00 | 0EA | 27 | 50 |
Up | 06.15 | 0EA | 27 | 50 |
Up | 06.30 | 0EA | 27 | 50 |
Up | 05.30 | 0C3 | 62 | 75 |
Up | 05.45 | 0C3 | 62 | 75 |
Up | 06.00 | 0C3 | 62 | 75 |
Up | 06.15 | 0C3 | 62 | 75 |
Up | 06.30 | 0C3 | 62 | 75 |
Up | 05.30 | 0BE | 27 | 50 |
Up | 05.45 | 0BE | 27 | 50 |
Up | 06.00 | 0BE | 27 | 50 |
Up | 06.15 | 0BE | 27 | 50 |
Up | 06.30 | 0BE | 27 | 50 |
The State shows if a node is Up or Down and can have only two values
The filename column is actually time
The vsn is the node id and is unique
the count parameters says that for a specific node at the specific time how many sensors are reading data
and the last column is just percentage of sensor reading data
I want to create a heatmap where I can choose the state from the drop down menu
If chosen Up it shows all the percentage (25,50,75 ) values by default
The heatmap x axis should be vsn and y axis should be the time(filename) and it should be colored based on count of parameters.
I have written this code
which gives me some sort of heatmap but is not what I am expecting
#! /usr/bin/env python
import pandas as pd
import numpy as np
import os
import glob
import datetime as dt
import warnings
import plotly.offline as pyo
import plotly.graph_objs as go
import plotly.figure_factory as ff
import dask.dataframe as ddimport dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Outputwarnings.filterwarnings(‘ignore’)
file_path=glob.glob( ‘/home/ricky/final/data/AoT*’)
for name in glob.glob(‘/home/ricky/final/data/AoT*’):
data_df=dd.read_csv(os.path.join(name, ‘data.csv’))
node_df=pd.read_csv(os.path.join(name, ‘nodes.csv’))
sensors_df=pd.read_csv(os.path.join(name, ‘sensors.csv’))
#data_df =data_df.compute()node_df[‘Status’] = node_df.apply(lambda x: ‘Active’ if pd.isnull(x[‘end_timestamp’]) else ‘Inactive’, axis=1)
node_df=node_df.loc[node_df.Status == ‘Active’]
node_df=node_df.drop(columns=[‘description’,‘project_id’,‘start_timestamp’,‘end_timestamp’])sensors_df=sensors_df.drop(columns=[‘ontology’,‘subsystem’,‘hrf_unit’,‘hrf_minval’,‘hrf_maxval’,‘datasheet’])
no_of_sensors=sensors_df[‘sensor’].nunique()
no_of_parameter=sensors_df[‘parameter’].nunique()#os.chdir(‘/home/ricky/final/data/’)
csvfiles=glob.glob( ‘/home/ricky/final/data/recent_data*.csv’)#csvfiles = glob.glob( ‘recent_data*.csv’)
dataframes = # a list to hold all the individual pandas DataFrames
for csvfile in csvfiles:
df = pd.read_csv(csvfile)
nd=pd.merge(df,node_df, on=‘node_id’, how=“outer”)
nd[‘filename’]=csvfile[35:40]
nd.fillna(‘unknown’, inplace=True)
dataframes.append(nd)concatenate them all together
data_node_join_df = pd.concat(dataframes, ignore_index=True)
data_node_join_df[‘State’] = data_node_join_df.apply(lambda x: ‘Down’ if (x[‘timestamp’]==‘unknown’) else ‘Up’, axis=1)heatmap_df=pd.DataFrame({‘Count_Parameters’: data_node_join_df.groupby((‘State’,‘filename’,‘vsn’)).parameter.nunique()}).reset_index()
heatmap_df.loc[heatmap_df[‘Count_Parameters’] <= 1, ‘Percentage_of_sensor_reading_data’] = 0
heatmap_df.loc[(heatmap_df[‘Count_Parameters’] > 1)
& (heatmap_df[‘Count_Parameters’]<=round(no_of_parameter*.25)), ‘Percentage_of_sensor_reading_data’] = 25heatmap_df.loc[(heatmap_df[‘Count_Parameters’]>round(no_of_parameter*.25))
& (heatmap_df[‘Count_Parameters’]<=round(no_of_parameter*.50)), ‘Percentage_of_sensor_reading_data’] = 50heatmap_df.loc[(heatmap_df[‘Count_Parameters’]>round(no_of_parameter*.50))
& (heatmap_df[‘Count_Parameters’]<=round(no_of_parameter*.75)), ‘Percentage_of_sensor_reading_data’] = 75heatmap_df.loc[(heatmap_df[‘Count_Parameters’]>round(no_of_parameter*.75))
& (heatmap_df[‘Count_Parameters’]<round(no_of_parameter)), ‘Percentage_of_sensor_reading_data’] = 99heatmap_df.loc[heatmap_df[‘Count_Parameters’] == no_of_parameter, ‘Percentage_of_sensor_reading_data’] = 100
heatmap_df[‘Percentage_of_sensor_reading_data’] = heatmap_df[‘Percentage_of_sensor_reading_data’].apply(np.int64)
heatmap_df=heatmap_df.sort_values(by=[‘State’,‘vsn’,‘Count_Parameters’],ascending=False)
#print(heatmap_df)data =
state = [“Up”,“Down”]
for i in state:
for j in range(100):
data.append([i,j])
check = pd.DataFrame(data,columns=[‘State’,‘Count_Parameters’])#states = heatmap_df[‘State’].unique()
#state_default = heatmap_df[‘State’][0]
#print(state_default)no_sensor_default = [50,75,25,]
#print(no_sensor_default)df_default = heatmap_df[(heatmap_df[‘State’] == ‘Up’) &
(heatmap_df[‘Percentage_of_sensor_reading_data’] != 0)]
#print(df_default)external_stylesheets = [‘https://codepen.io/chriddyp/pen/bWLwgP.css’]
app = dash.Dash(name, external_stylesheets=external_stylesheets)
app.layout = html.Div([
html.H1(‘Node Status’),
html.Div([
html.Div([
html.H4(‘Select Node State’),
dcc.Dropdown(
id=‘state_dropdown’,
options=[{‘label’: i, ‘value’: i} for i in heatmap_df[‘State’].unique()],
value = ‘Up’
),
],
style={‘width’: ‘48%’, ‘display’: ‘inline-block’}),html.Div([
html.H4(‘Select percentage of nodes reading data’),
dcc.Dropdown(
id=‘Menu_dropdown’,multi=True,
value = [50,75,25,]
),
],style={‘width’: ‘48%’, ‘float’: ‘left’, ‘display’: ‘inline-block’}),
dcc.Graph(id=‘heatmap’,
figure = {
‘data’: [go.Heatmap(
x=df_default[‘vsn’],
y=df_default[‘filename’],
z=df_default[‘Count_Parameters’],
hovertemplate=‘Depth: %{x}
Time: %{y}
Range: %{z}’,
name = ‘first legend group’,colorscale=‘YlOrRd’)],
‘layout’: go.Layout(
xaxis = dict(title = ‘vsn’,tickvals= df_default[‘vsn’].unique(), tickfont= {“size”: 8}, tickangle= -40,tickmode = ‘linear’),
yaxis = dict( title = ‘time’,tickvals= df_default[‘filename’].unique(), tickfont= {“size”: 8},tickmode = ‘linear’),
)})
]),])@app.callback(
dash.dependencies.Output(component_id=‘Menu_dropdown’,component_property=‘options’),
[dash.dependencies.Input(component_id=‘state_dropdown’,component_property=‘value’)]
)
def update_Menu_dropdown(selected_state):
return [{‘label’: i, ‘value’: i} for i in heatmap_df[heatmap_df[‘State’] == selected_state][‘Percentage_of_sensor_reading_data’].unique()]def update_graph(state_dropdown,Menu_dropdown):
heatmap_data = heatmap_df[(heatmap_df[‘State’] == state_dropdown) & (heatmap_df[‘Percentage_of_sensor_reading_data’] == Menu_dropdown)][[‘filename’,‘vsn’,‘Count_Parameters’]]
#heatmap_data = pd.merge(data, heatmap, on=[‘State’, ‘Count_Parameters’],how=‘outer’).fillna(0)
print (state_dropdown,Menu_dropdown)
maxsale = heatmap_data[heatmap_data[‘Count_Parameters’]==heatmap_data[‘Count_Parameters’].max()]
maxsale = maxsale.reset_index()
return {
‘data’: [go.Heatmap(
x=heatmap_data[‘vsn’],
y=heatmap_data[‘filename’],
z=heatmap_data[‘Count_Parameters’],
xgap = 3,
ygap = 3,
colorscale=‘YlOrRd’)],
‘layout’: go.Layout(
title = 'MAJORITY OF ‘+Menu_dropdown+’ SOLD AT ‘+str.upper(state_dropdown)+’ IS ON ‘+ str.upper(maxsale[‘vsn’][0])+’ '+str(maxsale[‘filename’][0])
)
}if name == ‘main’:
app.run_server(debug=True)
Can anyone please guide me what am I doing wrong?