From 47ef62411a8f3b6cfe21f763bddf50f55032ed3b Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Tue, 28 Jan 2025 15:27:06 -0800 Subject: [PATCH 1/7] Add TODO comments --- e3sm_diags/e3sm_diags_driver.py | 2 ++ e3sm_diags/viewer/main.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index b1726718c..83f46e257 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -392,6 +392,8 @@ def main(parameters=[]) -> List[CoreParameter]: # noqa B006 if not os.path.exists(path): os.makedirs(path) + # TODO: This is where the path is passed for the viewer, which + # is where the index.html is stored. index_path = create_viewer(path, parameters_results) logger.info("Viewer HTML generated at {}".format(index_path)) diff --git a/e3sm_diags/viewer/main.py b/e3sm_diags/viewer/main.py index 2410a877b..d1cbad7e1 100644 --- a/e3sm_diags/viewer/main.py +++ b/e3sm_diags/viewer/main.py @@ -77,6 +77,8 @@ def insert_data_in_row(row_obj, name, url): row_obj.append(td) path = os.path.join(e3sm_diags.INSTALL_PATH, "viewer", "index_template.html") + # TODO: root_dir is the results_dir with "viewer" appended to it. We + # want the index.html to work in the results_dir too. output = os.path.join(root_dir, "index.html") soup = BeautifulSoup(open(path), "lxml") From 83f1291b507d61b4d6522e8cf3371e68b9dd9c99 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Wed, 29 Jan 2025 16:41:23 -0600 Subject: [PATCH 2/7] Generate `index.html` at top-level of results directory --- .../562-index-html/562_index_html.cfg | 13 +++++++++++++ .../562-index-html/run_script.py | 11 +++++++++++ e3sm_diags/e3sm_diags_driver.py | 8 +------- e3sm_diags/viewer/core_viewer.py | 2 +- e3sm_diags/viewer/main.py | 10 +++++++++- 5 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 auxiliary_tools/cdat_regression_testing/562-index-html/562_index_html.cfg create mode 100644 auxiliary_tools/cdat_regression_testing/562-index-html/run_script.py diff --git a/auxiliary_tools/cdat_regression_testing/562-index-html/562_index_html.cfg b/auxiliary_tools/cdat_regression_testing/562-index-html/562_index_html.cfg new file mode 100644 index 000000000..30c76caa5 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/562-index-html/562_index_html.cfg @@ -0,0 +1,13 @@ +[#] +sets = ["polar"] +case_id = "GPCP_v3.2" +variables = ["PRECT"] +ref_name = "GPCP_v3.2" +reference_name = "GPCP v2.2" +seasons = ["ANN", "DJF", "MAM", "JJA", "SON"] +regions = ["polar_S"] +test_colormap = "WhiteBlueGreenYellowRed.rgb" +reference_colormap = "WhiteBlueGreenYellowRed.rgb" +diff_colormap = "BrBG" +contour_levels = [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5] +diff_levels = [-2, -1.5, -1, -0.75, -0.5, -0.25, 0.25, 0.5, 0.75, 1, 1.5, 2] diff --git a/auxiliary_tools/cdat_regression_testing/562-index-html/run_script.py b/auxiliary_tools/cdat_regression_testing/562-index-html/run_script.py new file mode 100644 index 000000000..a1e0a22cf --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/562-index-html/run_script.py @@ -0,0 +1,11 @@ +# python -m auxiliary_tools.cdat_regression_testing.792-lat-lon-run-script.792_lat_lon_run_script +from auxiliary_tools.cdat_regression_testing.base_run_script import run_set + +SET_NAME = "polar" +SET_DIR = "562-index-html" +# CFG_PATH: str | None = None +CFG_PATH: str | None = "auxiliary_tools/cdat_regression_testing/562-index-html/562_index_html.cfg" +MULTIPROCESSING = False + +# %% +run_set(SET_NAME, SET_DIR, CFG_PATH, MULTIPROCESSING) diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index 83f46e257..18f6eedfe 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -388,13 +388,7 @@ def main(parameters=[]) -> List[CoreParameter]: # noqa B006 if parameters_results[0].no_viewer: logger.info("Viewer not created because the no_viewer parameter is True.") else: - path = os.path.join(parameters_results[0].results_dir, "viewer") - if not os.path.exists(path): - os.makedirs(path) - - # TODO: This is where the path is passed for the viewer, which - # is where the index.html is stored. - index_path = create_viewer(path, parameters_results) + index_path = create_viewer(parameters_results) logger.info("Viewer HTML generated at {}".format(index_path)) # Validate actual and expected parameters are aligned diff --git a/e3sm_diags/viewer/core_viewer.py b/e3sm_diags/viewer/core_viewer.py index 1e29930b7..2bf3e9f65 100644 --- a/e3sm_diags/viewer/core_viewer.py +++ b/e3sm_diags/viewer/core_viewer.py @@ -100,7 +100,7 @@ def generate_page(self): ) return url - raise RuntimeError("Error geneating the page.") + raise RuntimeError("Error generating the page.") def generate_viewer(self, prompt_user=True): """Generate the webpage and ask the user if they want to see it.""" diff --git a/e3sm_diags/viewer/main.py b/e3sm_diags/viewer/main.py index d1cbad7e1..49330ab1b 100644 --- a/e3sm_diags/viewer/main.py +++ b/e3sm_diags/viewer/main.py @@ -1,10 +1,12 @@ import collections import os +from typing import List from bs4 import BeautifulSoup import e3sm_diags from e3sm_diags.logger import custom_logger +from e3sm_diags.parameter.core_parameter import CoreParameter from . import ( aerosol_budget_viewer, @@ -115,11 +117,16 @@ def insert_data_in_row(row_obj, name, url): return output -def create_viewer(root_dir, parameters): +def create_viewer(parameters: List[CoreParameter]) -> str: """ Based of the parameters, find the files with the certain extension and create the viewer in root_dir. """ + root_dir = parameters[0].results_dir + + if not os.path.exists(root_dir): + os.makedirs(root_dir) + # Group each parameter object based on the `sets` parameter. set_to_parameters = collections.defaultdict(list) for param in parameters: @@ -129,6 +136,7 @@ def create_viewer(root_dir, parameters): # A list of (title, url) tuples that each viewer generates. # This is used to create the main index. title_and_url_list = [] + # Now call the viewers with the list of parameters as the arguments. for set_name, parameters in set_to_parameters.items(): logger.info(f"{set_name} {root_dir}") From b3270671807a1c9b20adde1cad854b8a923249e0 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Wed, 29 Jan 2025 14:42:07 -0800 Subject: [PATCH 3/7] Update e3sm_diags/viewer/main.py --- e3sm_diags/viewer/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/e3sm_diags/viewer/main.py b/e3sm_diags/viewer/main.py index 49330ab1b..3fc2f752e 100644 --- a/e3sm_diags/viewer/main.py +++ b/e3sm_diags/viewer/main.py @@ -79,8 +79,6 @@ def insert_data_in_row(row_obj, name, url): row_obj.append(td) path = os.path.join(e3sm_diags.INSTALL_PATH, "viewer", "index_template.html") - # TODO: root_dir is the results_dir with "viewer" appended to it. We - # want the index.html to work in the results_dir too. output = os.path.join(root_dir, "index.html") soup = BeautifulSoup(open(path), "lxml") From 08971ced551bb3e02736783466637cf1982daa59 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Wed, 29 Jan 2025 17:54:59 -0600 Subject: [PATCH 4/7] Add top-level index.html to redirect to viewer index.html --- e3sm_diags/viewer/main.py | 44 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/e3sm_diags/viewer/main.py b/e3sm_diags/viewer/main.py index 3fc2f752e..e55d721bb 100644 --- a/e3sm_diags/viewer/main.py +++ b/e3sm_diags/viewer/main.py @@ -121,9 +121,10 @@ def create_viewer(parameters: List[CoreParameter]) -> str: certain extension and create the viewer in root_dir. """ root_dir = parameters[0].results_dir + viewer_dir = os.path.join(root_dir, "viewer") - if not os.path.exists(root_dir): - os.makedirs(root_dir) + if not os.path.exists(viewer_dir): + os.makedirs(viewer_dir) # Group each parameter object based on the `sets` parameter. set_to_parameters = collections.defaultdict(list) @@ -137,9 +138,9 @@ def create_viewer(parameters: List[CoreParameter]) -> str: # Now call the viewers with the list of parameters as the arguments. for set_name, parameters in set_to_parameters.items(): - logger.info(f"{set_name} {root_dir}") + logger.info(f"{set_name} {viewer_dir}") viewer_function = SET_TO_VIEWER[set_name] - result = viewer_function(root_dir, parameters) + result = viewer_function(viewer_dir, parameters) logger.info(result) title_and_url_list.append(result) @@ -147,7 +148,38 @@ def create_viewer(parameters: List[CoreParameter]) -> str: prov_tuple = ("Provenance", "../prov") title_and_url_list.append(prov_tuple) - index_url = create_index(root_dir, title_and_url_list) - utils.add_header(root_dir, index_url, parameters) + index_url = create_index(viewer_dir, title_and_url_list) + _create_root_index(root_dir, index_url) + + utils.add_header(viewer_dir, index_url, parameters) return index_url + + +def _create_root_index(root_dir: str, viewer_index_url: str): + """Create a root level `index.html` file that redirects to the viewer index. + + Parameters + ---------- + root_dir : str + The root directory. + index_url : str + The url to the viewer index.html file. + """ + root_index_path = os.path.join(root_dir, "index.html") + relative_viewer_index_url = os.path.relpath(viewer_index_url, root_dir) + root_soup = BeautifulSoup( + f""" + + + + + + + """, + "lxml", + ) + + # Write the root index file + with open(root_index_path, "wb") as f: + f.write(root_soup.prettify("utf-8")) From 47c37e8064bcd094aa8b7a784c7cb35eb9f1cf62 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Wed, 29 Jan 2025 17:59:29 -0600 Subject: [PATCH 5/7] Fix logic generating index.html for prov dir --- e3sm_diags/e3sm_diags_driver.py | 35 ++++++++++++++------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index 18f6eedfe..8f1a051e6 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -141,26 +141,21 @@ def save_provenance(results_dir, parser): if not os.path.exists(results_dir): os.makedirs(results_dir, 0o755) - # Create a PHP file to list the contents of the prov dir. - php_path = os.path.join(results_dir, "index.php") - with open(php_path, "w") as f: - contents = """ - $file

"; - $i++; - } - } - closedir($dh); - ?> - """ - f.write(contents) + # Create an HTML file to list the contents of the prov dir. + index_html_path = os.path.join(results_dir, "index.html") + + with open(index_html_path, "w") as f: + f.write("

Provenance Files

    ") + + for file_name in os.listdir(results_dir): + file_path = os.path.join(results_dir, file_name) + if os.path.isfile(file_path): + f.write(f'
  • {file_name}
    • ') + + f.write("
") + + logger.info("Created provenance index HTML file at: {}".format(index_html_path)) + try: _save_env_yml(results_dir) except Exception: From 20bfb8334dfd7e9bcaa211eab8a6920b73fcead2 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Wed, 29 Jan 2025 18:00:52 -0600 Subject: [PATCH 6/7] Open prov dir links in a new tab --- e3sm_diags/e3sm_diags_driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e3sm_diags/e3sm_diags_driver.py b/e3sm_diags/e3sm_diags_driver.py index 8f1a051e6..e3e8f1649 100644 --- a/e3sm_diags/e3sm_diags_driver.py +++ b/e3sm_diags/e3sm_diags_driver.py @@ -150,7 +150,9 @@ def save_provenance(results_dir, parser): for file_name in os.listdir(results_dir): file_path = os.path.join(results_dir, file_name) if os.path.isfile(file_path): - f.write(f'
  • {file_name}
    • ') + f.write( + f'
  • {file_name}
    • ' + ) f.write("") From 3bfe84235440e7295f69e8abaf48dd68504da6b1 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Wed, 29 Jan 2025 18:02:29 -0600 Subject: [PATCH 7/7] Make viewer html links open in new tabs --- e3sm_diags/viewer/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e3sm_diags/viewer/main.py b/e3sm_diags/viewer/main.py index e55d721bb..a813b788e 100644 --- a/e3sm_diags/viewer/main.py +++ b/e3sm_diags/viewer/main.py @@ -74,6 +74,7 @@ def insert_data_in_row(row_obj, name, url): td = soup.new_tag("td") a = soup.new_tag("a") a["href"] = url + a["target"] = "_blank" # Open link in a new tab a.string = name td.append(a) row_obj.append(td)