diff --git a/dmrpp_generator/generate_and_validate_dmrpp.py b/dmrpp_generator/generate_and_validate_dmrpp.py index f36a3c9..99e6837 100755 --- a/dmrpp_generator/generate_and_validate_dmrpp.py +++ b/dmrpp_generator/generate_and_validate_dmrpp.py @@ -42,7 +42,7 @@ def generate_docker_compose(): # Path to dockerfile. # '.' represents the current directory in which # docker-compose.yml is present. - image: ghrcdaac/dmrpp-generator:v3.4.0 + image: ghrcdaac/dmrpp-generator:v4.1.1 environment: - PAYLOAD=${PAYLOAD} # Mount volume diff --git a/dmrpp_generator/main.py b/dmrpp_generator/main.py index c3af65a..6c69cd4 100644 --- a/dmrpp_generator/main.py +++ b/dmrpp_generator/main.py @@ -147,7 +147,7 @@ def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False dmrpp_meta = dmrpp_meta if isinstance(dmrpp_meta, dict) else {} dmrpp_options = DMRppOptions(self.path) options = dmrpp_options.get_dmrpp_option(dmrpp_meta=dmrpp_meta) - local_option = f"-u file://{output_filename}" if local else "" + local_option = f"-u file://{output_filename}" if '-u' in options else '' dmrpp_cmd = f"get_dmrpp {options} {input_path} -o {output_filename}.dmrpp" \ f" {local_option} {os.path.basename(output_filename)}" return " ".join(dmrpp_cmd.split()) @@ -174,7 +174,11 @@ def run_command(self, cmd): stdout = subprocess.PIPE stderr = subprocess.STDOUT - out = subprocess.run(cmd.split(), stdout=stdout, stderr=stderr, timeout=self.timeout, check=True) + try: + out = subprocess.run(cmd.split(), stdout=stdout, stderr=stderr, timeout=self.timeout, check=True) + except Exception as e: + self.logger_to_cw.info(f'cmd: {cmd}') + raise Exception(f'get_dmrpp failed. \ncmd: {cmd} \nException: {e}') return out diff --git a/generate_and_validate_dmrpp b/generate_and_validate_dmrpp deleted file mode 100755 index 9d6fc3a..0000000 --- a/generate_and_validate_dmrpp +++ /dev/null @@ -1,114 +0,0 @@ -#! /usr/bin/python3 -import argparse -import subprocess -import time -from multiprocessing import Process -from os import walk -import tempfile - - -def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, separate_bar='-', length=100, fill='█', - print_end="\r"): - """ - Call in a loop to create terminal progress bar - @params: - iteration - Required : current iteration (Int) - total - Required : total iterations (Int) - prefix - Optional : prefix string (Str) - suffix - Optional : suffix string (Str) - decimals - Optional : positive number of decimals in percent complete (Int) - separate_bar - Optional : what will separate the bar as it fills - length - Optional : character length of bar (Int) - fill - Optional : bar fill character (Str) - printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) - """ - percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) - filled_length = int(length * iteration // total) - bar = fill * filled_length + separate_bar * (length - filled_length) - print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=print_end) - # Print New Line on Complete - if iteration == total: - print() - - -def progress_bar(file_number, prefix='Generating:', suffix='Complete', length=50, fill='█', separate_bar='-'): - items = list(range(0, min(file_number * 25, 600))) - l = len(items) - # Initial call to print 0% progress - print_progress_bar(iteration=0, total=l, prefix=prefix, suffix=suffix, length=length, fill=fill, - separate_bar=separate_bar) - for i, _ in enumerate(items): - time.sleep(0.1) - # Update Progress Bar - print_progress_bar(iteration=i + 1, total=l, prefix=prefix, suffix=suffix, length=length, fill=fill, - separate_bar=separate_bar) - - -def check_docker_version(log_file_path): - with open(log_file_path, "a+") as output: - dkr_comp_version = 'docker compose' - subprocess.run(f"{dkr_comp_version} version", shell=True, check=False, stdout=output, stderr=output) - output.seek(0) - err_grab = output.readlines()[-1] - if err_grab == f'/bin/sh: 1: {dkr_comp_version}: not found\n': - dkr_comp_version = 'docker-compose' - return dkr_comp_version - - -def run_docker_compose(payload, nc_hdf_path, port, dmrrpp_service, log_file_path): - dkr_comp_version = check_docker_version(log_file_path) - cmd = f"PAYLOAD='{payload}' NC_FILES_PATH={nc_hdf_path} PORT={port} {dkr_comp_version} up {dmrrpp_service}" - with open(log_file_path, "a") as output: - try: - subprocess.run( - cmd, - shell=True, check=False, stdout=output, - stderr=output) - - except KeyboardInterrupt: - subprocess.run(f" {dkr_comp_version} down {dmrrpp_service}", shell=True, check=False, stdout=output, - stderr=output) - - -def main(): - parser = argparse.ArgumentParser(description='Generate and validate DMRPP files.') - parser.add_argument('-p', '--path', dest='nc_hdf_path', nargs=1, required=True, - help='Path to netCDF4 and HDF5 folder') - parser.add_argument('-prt', '--port', dest='port', nargs=1, default=["8080"], - help='Port number to Hyrax local server') - parser.add_argument('-pyld', '--payload', dest='payload', nargs=1, default=['{}'], - help='Payload to execute get_dmrpp binary') - parser.add_argument('-vldt', '--validate', dest='validate', nargs=1, default=['true'], - help='Validate netCDF4 and HDF5 files against OPeNDAP local server') - - args = parser.parse_args() - nc_hdf_path, port, payload, validate = [getattr(args, ele)[0] for ele in args.__dict__.keys()] - no_need_validation = validate not in ['true', '1', 'yes', 'y'] - # If the user doesn't want validation run dmrpp service alone without Hyrax UI - dmrrpp_service = "dmrpp" if no_need_validation else "" - # Depending on needing the validation the user should get either a path or Hyrax UI link - visit_link_path_message = f"{nc_hdf_path}" if no_need_validation else f"http://localhost:{port}/opendap (^C to kill the server)" - - # Counting number of files to estimate the work - _, _, files = next(walk(nc_hdf_path)) - # Remove dmrpp suffix from the list of files - [files.remove(file_) for file_ in files[:] if file_.endswith('.dmrpp')] - _, log_file_location = tempfile.mkstemp(prefix="dmrpp-generator-") - - try: - p_1 = Process(target=progress_bar, args=(len(files),)) - p_2 = Process(target=run_docker_compose, args=(payload, nc_hdf_path, port, dmrrpp_service, log_file_location)) - p_1.start() - p_2.start() - p_1.join() - print(f"To see the results visit:\t{visit_link_path_message}\nLogs are located here:\t{log_file_location}") - p_2.join() - except KeyboardInterrupt: - print("Shutting down the server...") - p_1 = Process(target=progress_bar, args=(3, "Progress", 'Complete', 10, '💀', '🔥',)) - p_1.start() - p_1.join() - - -if __name__ == "__main__": - main()