diff --git a/Dockerfile b/Dockerfile index 2382e28..9762469 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /app ADD . /app RUN pip install -r requirements.txt RUN git clone https://github.com/sdgtt/telemetry.git -RUN cd telemetry && pip install -r requirements.txt && python setup.py install && cd .. +RUN cd telemetry && pip install -r requirements.txt && pip install . && cd .. RUN chmod u+x ./entrypoint.sh ENTRYPOINT ["./entrypoint.sh"] EXPOSE 5000 diff --git a/README.md b/README.md index 084b86b..708ed7e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,35 @@ # constellation -Web UI for testing +Constellation is a landing page to display all of the results of Kuiper Linux testing on hardware. + +Requirements on running: + +cloned constellation +cloned telemetry +virtual environment of constellation + + +--> Constellation setup <-- +(Recommended: Run to vm) +1. Create owner folder or go to your preferred folder in your linux server. + mkrdir or cd +2. Clone Constellation and Telemetry + git clone https://github.com/sdgtt/constellation.git + git clone https://github.com/sdgtt/telemetry.git +3. Create virtual environment, to create type this to your terminal: + python3 -m virtualenv +4. Go to Telemetry cloned folder and install requirements + cd telemetry/ + pip install -r requirements.txt + pip install -r requirements_dev.txt + cd .. +5. Run or activate your virtualenv + source envConstellation/bin/activate + +6. Go to Constellation cloned folder and install requirements + pip install telemetry + pip install -r requirements.txt + pip install -r requirements_dev.txt +7. Once installed, run the in development mode. + export FLASK_ENV=development + export ES=192.168.10.12 + flask run --host=0.0.0.0 --port=5002 diff --git a/app/__init__.py b/app/__init__.py index b9f1220..1507b3a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -24,7 +24,6 @@ def register_dashapps(server): "name": "viewport", "content": "width=device-width, initial-scale=1, shrink-to-fit=no", } - score_card = Dash( __name__, server=server, diff --git a/app/app.py b/app/app.py index 89b5d12..645d818 100644 --- a/app/app.py +++ b/app/app.py @@ -1,3 +1,4 @@ +import os from urllib.parse import unquote, urlparse from app.models import boards as b @@ -7,7 +8,13 @@ from app.pages.hwtests import allboards as ab from app.pages.publicci.dashboard import Dashboard from app.pages.pyadi.plots import gen_line_plot_html -from app.utility import artifact_url_gen, filter_gen, url_gen +from app.utility import ( + append_url_to_dict, + artifact_url_gen, + filter_gen, + result_json_url, + url_gen, +) from flask import ( Blueprint, Flask, @@ -26,7 +33,9 @@ # app = Flask(__name__) server_bp = Blueprint("constellation", __name__) -JENKINS_SERVER = "gateway.englab" +JENKINS_SERVER = ( + "jenkinsci" if "JENKINS_SERVER" not in os.environ else os.environ["JENKINS_SERVER"] +) JENKINS_PORT = None pci_dash = Dashboard( @@ -77,7 +86,7 @@ def api(param=None): else: kwargs.update({f: el}) result_json = DB().search(size=size, order=order, agg_field=agg_field, **kwargs) - return result_json + return result_json_url(result_json, jenkins_url=JENKINS_SERVER) @server_bp.route("api/board//") @@ -96,7 +105,9 @@ def board_api(board_name, param=None): for k, v in boot_test.items(): if k not in ["boot_test_failure", "raw_boot_test_result"]: new_dict.update({k: v}) - boot_test_filtered.append(new_dict) + boot_test_filtered.append( + append_url_to_dict(new_dict, jenkins_url=JENKINS_SERVER) + ) return {"hits": boot_test_filtered} @@ -104,9 +115,9 @@ def board_api(board_name, param=None): @server_bp.route("api/sc/") def score_api(param=None): default_jenkins_project = "HW_tests/HW_test_multiconfig" - default_branch = "boot_partition_master" + default_branch = "" default_size = 7 - default_offset = 0 + default_offset = 0 filters = filter_gen(urlparse(unquote(request.url)).query) jenkins_project = ( filters["jenkins_project"][0] @@ -124,7 +135,7 @@ def score_api(param=None): branch=branch, board=board, deprecated=deprecated, - offset = int(offset) + offset=int(offset), ) return sc.to_json() @@ -134,7 +145,7 @@ def allboards(): # retrieve boards from elastic server # filter by jenkins_project_name jenkins_project_name = "HW_tests/HW_test_multiconfig" - source_adjacency_matrix = "boot_partition_master" + source_adjacency_matrix = "boot_partition_main" deprecated = [] boards_ref = b.Boards( jenkins_project_name, source_adjacency_matrix, deprecated diff --git a/app/models/artifacts.py b/app/models/artifacts.py index 66baa83..b326ddc 100644 --- a/app/models/artifacts.py +++ b/app/models/artifacts.py @@ -9,7 +9,7 @@ class Artifact: "job_build_parameters", "file_name", "target_board", - "artifact_info_type" + "artifact_info_type", ] def __init__(self, raw_artifact_result=None): @@ -48,7 +48,7 @@ def display(self): class Artifacts: def __init__(self, **filters): - self.db = DB(index_name="artifacts",keywords=Artifact.KEYWORDS) + self.db = DB(index_name="artifacts", keywords=Artifact.KEYWORDS) db_res = self.db.search(sort="archive_date", **filters) # create boards object from raw db_res self._artifacts = [Artifact(row) for row in db_res["hits"]] diff --git a/app/models/boot_tests.py b/app/models/boot_tests.py index ae4472e..0f67f88 100644 --- a/app/models/boot_tests.py +++ b/app/models/boot_tests.py @@ -194,7 +194,6 @@ def latest_builds(self, size, offset=0): } ) return builds_dict - def latest_builds_boards(self, size, offset=0): boards = [] diff --git a/app/models/db.py b/app/models/db.py index e1df312..000bad9 100644 --- a/app/models/db.py +++ b/app/models/db.py @@ -23,6 +23,7 @@ "last_failing_stage", ] + class DB: def __init__( self, @@ -30,7 +31,7 @@ def __init__( username=USERNAME, password=PASSWORD, index_name=INDEX, - keywords=KEYWORDS + keywords=KEYWORDS, ): retries = 0 while True: @@ -42,7 +43,7 @@ def __init__( password=password, index_name=index_name, ) - self.keywords=keywords + self.keywords = keywords break except Exception as e: print("Server not yet ready") @@ -57,7 +58,7 @@ def search( sort="jenkins_job_date", order="desc", agg_field=None, - **kwargs + **kwargs, ): result = {"hits": [], "aggregates": [], "aggregates_top": []} if kwargs: @@ -85,11 +86,15 @@ def search( if agg_field: for row in res["aggregations"][agg_field]["buckets"]: result["aggregates"].append(row["key"]) - + agg_field_data = {} # get top data for the aggregated field for row_data in result["hits"]: - agg_field = agg_field.split(".keyword")[0] if "keyword" in agg_field else agg_field + agg_field = ( + agg_field.split(".keyword")[0] + if "keyword" in agg_field + else agg_field + ) if row_data[agg_field] == row["key"]: agg_field_data = row_data break @@ -114,11 +119,12 @@ def test_db_search(): elastic_server="primary.englab", username="", password="", - index_name="boot_tests" + index_name="boot_tests", ) result = db.search( size=1, sort="jenkins_job_date", order="desc", agg_field="boot_folder_name.keyword", - ) \ No newline at end of file + ) + assert result diff --git a/app/models/score.py b/app/models/score.py index 38da49a..27e1b34 100644 --- a/app/models/score.py +++ b/app/models/score.py @@ -27,7 +27,7 @@ def __init__( self, jenkins_project="HW_tests/HW_test_multiconfig", size=2, - branch="boot_partition_master", + branch="boot_partition_main", board=None, deprecated=[], offset=0, @@ -200,7 +200,9 @@ def get_test_results(self): # noqa: C901 ]: if isinstance(report[bn][test]["data"], list): try: - assert report[bn][test]["count"] == len(report[bn][test]["data"]) + assert report[bn][test]["count"] == len( + report[bn][test]["data"] + ) except Exception as e: print(f"Inconsistency in build {bn} on test {test}") print(report[bn][test]) @@ -210,7 +212,7 @@ def get_test_results(self): # noqa: C901 for k, boards in report[bn][test]["data"].items(): for board in boards: count += 1 - try: + try: assert report[bn][test]["count"] == count except Exception as e: print(f"Inconsistency in build {bn} on test {test}") @@ -361,7 +363,7 @@ def to_dict(self): if __name__ == "__main__": - sc = Score(size=7, branch="boot_partition_master") + sc = Score(size=7, branch="boot_partition_main") # print(sc.boot_tests) # print(sc.boards) # print(sc.builds) diff --git a/app/score_card/app.py b/app/score_card/app.py index d1fa6f5..299c680 100644 --- a/app/score_card/app.py +++ b/app/score_card/app.py @@ -3,12 +3,12 @@ from datetime import date, datetime, timedelta +import dash import dash_bootstrap_components as dbc import dash_dangerously_set_inner_html import pandas as pd import plotly.express as px import plotly.graph_objects as go -import dash from dash import Input, Output, State, dash_table, dcc, html from flask import Markup @@ -139,14 +139,9 @@ def get_fig_logs(data): ] ) fig.update_layout( - barmode="stack", + barmode="stack", title="Boot Logs Turnout", - legend=dict( - yanchor="top", - y=0.99, - xanchor="left", - x=0.01 - ), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), margin=dict(l=20, r=20), ) return fig @@ -179,14 +174,9 @@ def get_fig_booted(data): ] ) fig.update_layout( - barmode="stack", + barmode="stack", title="Boot Histogram", - legend=dict( - yanchor="top", - y=0.99, - xanchor="left", - x=0.01 - ), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), margin=dict(l=20, r=20), ) return fig @@ -213,14 +203,9 @@ def get_fig_linux(data): ] ) fig.update_layout( - barmode="stack", + barmode="stack", title="IIO Devices", - legend=dict( - yanchor="top", - y=0.99, - xanchor="left", - x=0.01 - ), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), margin=dict(l=20, r=20), ) return fig @@ -247,14 +232,9 @@ def get_fig_dmesg(data): ] ) fig.update_layout( - barmode="stack", + barmode="stack", title="DMESG Errors", - legend=dict( - yanchor="top", - y=0.99, - xanchor="left", - x=0.01 - ), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), margin=dict(l=20, r=20), ) return fig @@ -295,14 +275,9 @@ def get_fig_pyadi(data): ] ) fig.update_layout( - barmode="stack", + barmode="stack", title="PYADI-IIO Tests", - legend=dict( - yanchor="top", - y=0.99, - xanchor="left", - x=0.01 - ), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), margin=dict(l=20, r=20), ) return fig @@ -320,10 +295,11 @@ def generate_logo(): html.H3("Kuiper Linux CT Score Card"), ], id="sc_logo_div", - style={}, + style={"font-size": "15px"}, ) return sc_logo_div + def generate_filters_offcanvas(data): offcanvas = html.Div( [ @@ -338,32 +314,36 @@ def generate_filters_offcanvas(data): style={ "position": "absolute", "right": "0%", - "bottom": "0%" - } + "bottom": "0%", + "font-size": "15px", + }, ) return offcanvas + def generate_header(data): header_div = html.Div( [ dbc.Row( [ - dbc.Col(generate_logo(), width=10), + # dbc.Col(generate_logo(), width=10), dbc.Col(generate_filters_offcanvas(data), width=2), ], align="start", ), ], style={ - "margin-left":"10%", - "margin-right":"10%", + "margin-left": "2%", + "margin-right": "2%", "margin-top": "3%", "position": "relative", - } + "font-size": "15px", + }, ) return header_div + def generate_options(data): options_div = html.Div( @@ -374,7 +354,7 @@ def generate_options(data): type="text", placeholder="Jenkins project name", value="HW_tests/HW_test_multiconfig", - style={"width": "100%"}, + style={"width": "100%", "font-size": "15px"}, ), html.H5("Branch"), dcc.Dropdown( @@ -408,7 +388,7 @@ def generate_options(data): max=10, placeholder="No. of builds to Analyze", value=7, - style={"width": "100%"}, + style={"width": "100%", "font-size": "15px"}, ), html.H5("Offset"), dcc.Input( @@ -418,7 +398,7 @@ def generate_options(data): max=20, placeholder="No. of builds from the latest to analyze", value=0, - style={"width": "100%"}, + style={"width": "100%", "font-size": "15px"}, ), html.H5("Board"), dcc.Dropdown( @@ -484,13 +464,10 @@ def generate_dash_table(data, target, groupby="item"): style_cell={ "textAlign": "left", "vertical-align": "top", - "font-family":"var(--bs-body-font-family)", - "font-size":"var(--bs-body-font-size)", "font-weight": "var(--bs-body-font-weight)", }, style_header={ - "backgroundColor": "#D3D3D3", - "color": "black", + "color": "3e3e3e", }, data=data_processed, columns=cols, @@ -501,24 +478,27 @@ def generate_dash_table(data, target, groupby="item"): def generate_top_boot_failing(data): tabs = dbc.Tabs( - [ - dbc.Tab( - generate_dash_table(data, "not_booted", "board"), - tab_id="tab-1", - label="Failing Boards (Non Booting)", - ), - dbc.Tab(generate_dash_table(data, "not_booted"), - tab_id="tab-2", - label="Failures"), - ], - active_tab="tab-1", - ) + [ + dbc.Tab( + generate_dash_table(data, "not_booted", "board"), + tab_id="tab-1", + label="Failing Boards (Non Booting)", + ), + dbc.Tab( + generate_dash_table(data, "not_booted"), + tab_id="tab-2", + label="Failures", + ), + ], + active_tab="tab-1", + ) style = {"margin-top": "2%", "margin-bottom": "2%"} top_boot_failing_div = dbc.Row( children=[ dbc.Col(tabs, style=style, width=8), dbc.Col( - dcc.Graph(id="g_boot_test_summary", figure=get_fig_booted(data)),width=4 + dcc.Graph(id="g_boot_test_summary", figure=get_fig_booted(data)), + width=4, ), ], id="top_boot_failing_div", @@ -528,26 +508,27 @@ def generate_top_boot_failing(data): def generate_drivers_enumeration(data): tabs = dbc.Tabs( - [ - dbc.Tab( - generate_dash_table(data, "drivers_missing", "board"), - tab_id="tab-1", - label="Failing Boards (Failed Device Enumeration)", - ), - dbc.Tab( - generate_dash_table(data, "drivers_missing"), - tab_id="tab-2", - label="Missing Drivers", - ), - ], - active_tab="tab-1" - ) + [ + dbc.Tab( + generate_dash_table(data, "drivers_missing", "board"), + tab_id="tab-1", + label="Failing Boards (Failed Device Enumeration)", + ), + dbc.Tab( + generate_dash_table(data, "drivers_missing"), + tab_id="tab-2", + label="Missing Drivers", + ), + ], + active_tab="tab-1", + ) style = {"margin-top": "2%", "margin-bottom": "2%"} drivers_enumeration_div = dbc.Row( children=[ dbc.Col(tabs, style=style, width=8), dbc.Col( - dcc.Graph(id="g_linux_test_summary", figure=get_fig_linux(data)),width=4 + dcc.Graph(id="g_linux_test_summary", figure=get_fig_linux(data)), + width=4, ), ], id="drivers_enumeration_div", @@ -569,14 +550,15 @@ def generate_dmesg_errors(data): label="Dmeg Errors", ), ], - active_tab="tab-1" + active_tab="tab-1", ) style = {"margin-top": "2%", "margin-bottom": "2%"} dmesg_errors_div = dbc.Row( children=[ dbc.Col(tabs, style=style, width=8), dbc.Col( - dcc.Graph(id="g_linux_dmesg_summary", figure=get_fig_dmesg(data)),width=4 + dcc.Graph(id="g_linux_dmesg_summary", figure=get_fig_dmesg(data)), + width=4, ), ], id="dmesg_errors_div", @@ -603,15 +585,14 @@ def generate_pytest_results(data): tab_id="tab-3", label="PyADI IIO Skipped", ), - ], active_tab="tab-1" + ], + active_tab="tab-1", ) style = {"margin-top": "2%", "margin-bottom": "2%"} pyadi_test_div = dbc.Row( children=[ dbc.Col(tabs, style=style, width=8), - dbc.Col( - dcc.Graph(id="g_pyadi_test", figure=get_fig_pyadi(data)),width=4 - ), + dbc.Col(dcc.Graph(id="g_pyadi_test", figure=get_fig_pyadi(data)), width=4), ], id="pyadi_test_div", ) @@ -656,7 +637,8 @@ def generate_panel(data): dbc.Col( [ dbc.Row(dcc.Graph(id="g_logs_turnout", figure=get_fig_logs(data))), - ], width=4 + ], + width=4, ), ], justify="between", @@ -664,43 +646,53 @@ def generate_panel(data): ) return sc_panel_div -def generate_report(data): - report_div = generate_summaries(data) - return report_div def report_tabs(data, active_tab="t_summary"): - tabs = dbc.Tabs([ - dbc.Tab(label='Summaries', tab_id="t_summary", children=[ - generate_panel(data) - ]), - dbc.Tab(label='Boot Test', tab_id="t_boot_test", children=[ - generate_top_boot_failing(data), - ]), - dbc.Tab(label='Linux Test - IIO Drivers', tab_id="t_iio", children=[ - generate_drivers_enumeration(data) - ]), - dbc.Tab(label='Linux Test - DMESG', tab_id="t_dmesg", children=[ - generate_dmesg_errors(data), - ]), - dbc.Tab(label='PYADI-IIO Tests', tab_id="t_pyadi", children=[ - generate_pytest_results(data) - ]), - ], - active_tab=active_tab, - id="sc_report_tab" - ) + tabs = dbc.Tabs( + [ + dbc.Tab( + label="Summaries", tab_id="t_summary", children=[generate_panel(data)] + ), + dbc.Tab( + label="Boot Test", + tab_id="t_boot_test", + children=[ + generate_top_boot_failing(data), + ], + ), + dbc.Tab( + label="Linux Test - IIO Drivers", + tab_id="t_iio", + children=[generate_drivers_enumeration(data)], + ), + dbc.Tab( + label="Linux Test - DMESG", + tab_id="t_dmesg", + children=[ + generate_dmesg_errors(data), + ], + ), + dbc.Tab( + label="PYADI-IIO Tests", + tab_id="t_pyadi", + children=[generate_pytest_results(data)], + ), + ], + active_tab=active_tab, + id="sc_report_tab", + ) return tabs + def generate_report_tabs(data): return html.Div( id="sc_report_div", - style = {"width": "80%", "margin": "auto"}, - children=[ - report_tabs(data) - ] + style={"margin-left": "2%", "margin-right": "2%"}, + children=[report_tabs(data)], ) + ########################### Callbacks ##################################### @@ -713,13 +705,15 @@ def register_callbacks(app): Input("sc_size_options", "value"), Input("sc_board_options", "value"), Input("sc_offset_options", "value"), - Input("sc_report_tab","active_tab"), + Input("sc_report_tab", "active_tab"), ], ) def update_report(project, branch, size, board, offset, active_tab): return report_tabs( - request_data(project=project, branch=branch, size=size, board=board, offset=offset), - active_tab=active_tab + request_data( + project=project, branch=branch, size=size, board=board, offset=offset + ), + active_tab=active_tab, ) @app.callback( @@ -741,5 +735,5 @@ def toggle_offcanvas(n1, is_open): generate_report_tabs(request_data()), ], id="main_panel", - style={} -) \ No newline at end of file + style={"font-size": "15px"}, +) diff --git a/app/static/gco.svg b/app/static/gco.svg index 3282324..1b4270c 100644 --- a/app/static/gco.svg +++ b/app/static/gco.svg @@ -23,4 +23,4 @@ - \ No newline at end of file + diff --git a/app/static/style.css b/app/static/style.css index a173f00..48cb511 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1,3 +1,14 @@ +@import url('https://fonts.googleapis.com/css?family=Poppins'); + +body{ + font-family: 'Poppins', sans-serif; + color: #3e3e3e; + background-color: #FFFFFF; + width: 100%; + height: 100vh; + position: relative; + font-size: 15px; +} .path { stroke-dasharray: 1000; stroke-dashoffset: 1000; diff --git a/app/utility.py b/app/utility.py index fe81dae..8d401c2 100644 --- a/app/utility.py +++ b/app/utility.py @@ -116,3 +116,56 @@ def filter_gen(query): field_value = field.split("=")[1] filter_dict.update({field_name: field_value.split(",")}) return filter_dict + + +def append_url_to_dict( + boot_test_dict, + jenkins_url="localhost", + jenkins_port=None, + jenkins_base_path="/jenkins", +): + # print(boot_test_dict) + url_dict = url_gen( + jenkins_server=jenkins_url, + jenkins_port=jenkins_port, + jenkins_base_path=jenkins_base_path, + project_name=boot_test_dict["jenkins_project_name"], + build_number=boot_test_dict["jenkins_build_number"], + board=boot_test_dict["boot_folder_name"], + hdl_commit=boot_test_dict["hdl_hash"].split()[0], + linux_commit=boot_test_dict["linux_hash"].split()[0], + trigger=boot_test_dict["jenkins_trigger"], + ) + artifacts_url_dict = artifact_url_gen( + jenkins_server=jenkins_url, + jenkins_port=jenkins_port, + project_name=boot_test_dict["jenkins_project_name"], + build_number=boot_test_dict["jenkins_build_number"], + board=boot_test_dict["boot_folder_name"], + jenkins_base_path=jenkins_base_path, + ) + url_dict.update(artifacts_url_dict) + for field, url_value in url_dict.items(): + boot_test_dict.update({field.lower().replace(" ", "_") + "_url": url_value}) + return boot_test_dict + + +def result_json_url( + result_json, + jenkins_url="localhost", + jenkins_port=None, + jenkins_base_path="/jenkins", +): + new_result_json = {} + for k, v in result_json.items(): + if k in ["hits"]: + new_v = [] + for el in v: + new_el = append_url_to_dict( + el, jenkins_url, jenkins_port, jenkins_base_path + ) + new_v.append(new_el) + new_result_json[k] = new_v + else: + new_result_json[k] = v + return new_result_json