From c7b3b8edb62a9b26c8d6a124023cb20021a5581c Mon Sep 17 00:00:00 2001 From: Ivo Branco Date: Tue, 21 Jan 2025 16:15:01 +0000 Subject: [PATCH] fix: lint --- .../management/commands/export_course.py | 64 ++++++-- .../commands/export_course_content_async.py | 31 +++- .../transfer_export_course_content.py | 137 +++++++++++++----- 3 files changed, 177 insertions(+), 55 deletions(-) diff --git a/nau_openedx_extensions/studio/management/commands/export_course.py b/nau_openedx_extensions/studio/management/commands/export_course.py index 4b27531..0907b4b 100644 --- a/nau_openedx_extensions/studio/management/commands/export_course.py +++ b/nau_openedx_extensions/studio/management/commands/export_course.py @@ -1,6 +1,7 @@ """ This command has 2 execution modes, one for exporting course content and another for transferring course content. -It dispatch celery tasks that or export to a tar.gz file or transfer course content and upload it to the course 'GRADES_DOWNLOAD' storage. +It dispatch celery tasks that or export to a tar.gz file or transfer course content and upload it to the course +'GRADES_DOWNLOAD' storage. Warn: this django command will dispatch multiple celery tasks, making the CMS celery workers having many pending tasks. Please check the CMS_URL/heartbeat?extended to monitor its progress. @@ -15,11 +16,12 @@ python manage.py cms export_course --task export --username Transfer all courses: - python manage.py cms export_course --task transfer --username + python manage.py cms export_course --task transfer --username Transfer a specific course: python manage.py cms export_course --task transfer --username ,, """ + from time import sleep from cms.djangoapps.contentstore.tasks import export_olx # lint-amnesty, pylint: disable=import-error @@ -27,35 +29,60 @@ from django.core.management.base import BaseCommand from xmodule.modulestore.django import modulestore -from nau_openedx_extensions.studio.contentstore.tasks import \ - transfer_course_content # lint-amnesty, pylint: disable=import-error +from nau_openedx_extensions.studio.contentstore.tasks import ( # lint-amnesty, pylint: disable=import-error + transfer_course_content, +) User = get_user_model() + def get_task_name(task_name): """ Generate the task givent its name. """ - if task_name == 'export': + if task_name == "export": # The upstream export course content task return export_olx - elif task_name == 'transfer': + elif task_name == "transfer": # The custom transfer course content task return transfer_course_content else: raise "Task not supported" + class Command(BaseCommand): """ This command has 2 execution modes, one for exporting course content and another for transferring course content. - It dispatch celery tasks that or export to a tar.gz file or transfer course content and upload it to the course 'GRADES_DOWNLOAD' storage. + It dispatch celery tasks that or export to a tar.gz file or transfer course content and upload it to the course + 'GRADES_DOWNLOAD' storage. """ def add_arguments(self, parser): - parser.add_argument("--username", type=str, help="The username of the user to export the course content") - parser.add_argument("--task", default='export', nargs='?', choices=['export', 'transfer'], help="Task to execute: export or transfer") - parser.add_argument("--index", type=int, default=0, help="Start index of the course ids to begin exporting") - parser.add_argument("course_ids", nargs="*", metavar="course_id", default=None, help="Course ids to export or if omitted, all courses will be exported") + parser.add_argument( + "--username", + type=str, + help="The username of the user to export the course content", + ) + parser.add_argument( + "--task", + default="export", + nargs="?", + choices=["export", "transfer"], + help="Task to execute: export or transfer", + ) + parser.add_argument( + "--index", + type=int, + default=0, + help="Start index of the course ids to begin exporting", + ) + parser.add_argument( + "course_ids", + nargs="*", + metavar="course_id", + default=None, + help="Course ids to export or if omitted, all courses will be exported", + ) def log_msg(self, msg): self.stdout.write(msg) @@ -85,7 +112,9 @@ def handle(self, *args, **options): course_celery_task_dict = {} for index in range(start_index, courses_count): course_id = course_ids[index] - self.log_msg(f"Enqueue task {task_name} - {index+1} of {courses_count} - {course_id}") + self.log_msg( + f"Enqueue task {task_name} - {index+1} of {courses_count} - {course_id}" + ) task = task_func.delay(user_id, course_id, "en") course_celery_task_dict[course_id] = task @@ -97,13 +126,18 @@ def handle(self, *args, **options): while not celery_task.ready(): self.log_msg(f"Waiting for task {celery_task} to complete...") sleep(5) - self.log_msg(f"Task {celery_task} for course {course_id} has finished with {'success' if celery_task.successful() else 'failure'}") + self.log_msg( + f"Task {celery_task} for course {course_id} has finished with " + f"{'success' if celery_task.successful() else 'failure'}" + ) if task.successful(): successfull_courses.append(course_id) else: failed_courses.append(course_id) - - self.log_msg(f"Tasks completed, successful count: {len(successfull_courses)}, failed, {len(failed_courses)}") + + self.log_msg( + f"Tasks completed, successful count: {len(successfull_courses)}, failed, {len(failed_courses)}" + ) for failed_course in failed_courses: self.log_msg(f"Failed course: {failed_course}") diff --git a/nau_openedx_extensions/studio/management/commands/export_course_content_async.py b/nau_openedx_extensions/studio/management/commands/export_course_content_async.py index 8f85a47..7b1f0dd 100644 --- a/nau_openedx_extensions/studio/management/commands/export_course_content_async.py +++ b/nau_openedx_extensions/studio/management/commands/export_course_content_async.py @@ -14,6 +14,7 @@ Export all courses: python manage.py cms export_course_content_async --username """ + from cms.djangoapps.contentstore.tasks import export_olx # lint-amnesty, pylint: disable=import-error from django.conf import settings from django.contrib.auth import get_user_model @@ -33,9 +34,24 @@ class Command(BaseCommand): """ def add_arguments(self, parser): - parser.add_argument("--username", type=str, help="The username of the user to export the course content") - parser.add_argument("--index", type=int, default=0, help="Start index of the course ids to begin exporting") - parser.add_argument("course_ids", nargs="*", metavar="course_id", default=None, help="Course ids to export or if omitted, all courses will be exported") + parser.add_argument( + "--username", + type=str, + help="The username of the user to export the course content", + ) + parser.add_argument( + "--index", + type=int, + default=0, + help="Start index of the course ids to begin exporting", + ) + parser.add_argument( + "course_ids", + nargs="*", + metavar="course_id", + default=None, + help="Course ids to export or if omitted, all courses will be exported", + ) def log_msg(self, msg): self.stdout.write(msg) @@ -60,7 +76,9 @@ def handle(self, *args, **options): courses_count = len(course_ids) for index in range(start_index, courses_count): course_id = course_ids[index] - self.log_msg(f"Exporting {index+1} of {courses_count} - exporting {course_id}") + self.log_msg( + f"Exporting {index+1} of {courses_count} - exporting {course_id}" + ) try: course_key = CourseKey.from_string(course_id) @@ -69,9 +87,7 @@ def handle(self, *args, **options): cms_root_url = SiteConfiguration.get_value_for_org( course_key.org, "CMS_ROOT_URL", settings.CMS_ROOT_URL ) - to_download_url = ( - f"{cms_root_url}/export/{course_id}" - ) + to_download_url = f"{cms_root_url}/export/{course_id}" self.log_msg( f"You can confirm the existence of the file on: {to_download_url}" ) @@ -79,5 +95,6 @@ def handle(self, *args, **options): self.log_msg(f"Error exporting course {course_id}: {e}") # print stacktrace and continue import traceback + self.log_msg(traceback.format_exc()) continue diff --git a/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py b/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py index c8503ac..25bb019 100644 --- a/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py +++ b/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py @@ -12,6 +12,7 @@ Export all courses: python manage.py cms transfer_export_course_content --username """ + import base64 import os @@ -33,14 +34,18 @@ User = get_user_model() -class MockRequest(): + +class MockRequest: def __init__(self, user): self.user = user + FILE_READ_CHUNK = 1024 # bytes -def upload_tar_gz_to_report_store(file, name, course_id, timestamp, config_name="GRADES_DOWNLOAD"): +def upload_tar_gz_to_report_store( + file, name, course_id, timestamp, config_name="GRADES_DOWNLOAD" +): """ Upload given file buffer as a tar.gz file using ReportStore. """ @@ -49,47 +54,70 @@ def upload_tar_gz_to_report_store(file, name, course_id, timestamp, config_name= report_name = "{course_prefix}_{name}_{timestamp_str}.tar.gz".format( course_prefix=course_filename_prefix_generator(course_id), name=name, - timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M") + timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M"), ) report_store.store(course_id, report_name, file) return report_name -def upload_tar_gz(file_name, name, course_key, timestamp, config_name="GRADES_DOWNLOAD"): +def upload_tar_gz( + file_name, name, course_key, timestamp, config_name="GRADES_DOWNLOAD" +): """ Upload a tar.gz using aws cli or ReportStore. - The ReportStore sometimes fails to upload some files, so we use aws cli as primary upload method. """ - bucket = settings.GRADES_DOWNLOAD.get('BUCKET') + The ReportStore sometimes fails to upload some files, so we use aws cli as primary upload method. + """ + bucket = settings.GRADES_DOWNLOAD.get("BUCKET") if bucket: - AWS_ACCESS_KEY_ID = settings.GRADES_DOWNLOAD.get('AWS_ACCESS_KEY_ID', settings.AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY = settings.GRADES_DOWNLOAD.get('AWS_SECRET_ACCESS_KEY', settings.AWS_SECRET_ACCESS_KEY) - AWS_S3_ENDPOINT_URL = settings.GRADES_DOWNLOAD.get('STORAGE_KWARGS', {}).get('endpoint_url', settings.AWS_S3_ENDPOINT_URL) + AWS_ACCESS_KEY_ID = settings.GRADES_DOWNLOAD.get( + "AWS_ACCESS_KEY_ID", settings.AWS_ACCESS_KEY_ID + ) + AWS_SECRET_ACCESS_KEY = settings.GRADES_DOWNLOAD.get( + "AWS_SECRET_ACCESS_KEY", settings.AWS_SECRET_ACCESS_KEY + ) + AWS_S3_ENDPOINT_URL = settings.GRADES_DOWNLOAD.get("STORAGE_KWARGS", {}).get( + "endpoint_url", settings.AWS_S3_ENDPOINT_URL + ) report_store = ReportStore.from_config(config_name) report_name = "{course_prefix}_{name}_{timestamp_str}.tar.gz".format( course_prefix=course_filename_prefix_generator(course_key), name=name, - timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M") + timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M"), ) - path = report_store.path_to(course_key, report_name, '') + path = report_store.path_to(course_key, report_name, "") import os import subprocess + my_env = os.environ.copy() my_env["AWS_ACCESS_KEY_ID"] = AWS_ACCESS_KEY_ID my_env["AWS_SECRET_ACCESS_KEY"] = AWS_SECRET_ACCESS_KEY - returncode = subprocess.call(['aws', f"--endpoint={AWS_S3_ENDPOINT_URL}", 's3', 'cp', file_name, f"s3://{bucket}/{path}"], env=my_env) + returncode = subprocess.call( + [ + "aws", + f"--endpoint={AWS_S3_ENDPOINT_URL}", + "s3", + "cp", + file_name, + f"s3://{bucket}/{path}", + ], + env=my_env, + ) if returncode != 0: raise Exception(f"Failed to upload file to S3. Return code: {returncode}") - else: + else: with open(file_name, mode="r", encoding="utf-8") as file: - upload_tar_gz_to_report_store(file, name, course_key, timestamp, config_name) + upload_tar_gz_to_report_store( + file, name, course_key, timestamp, config_name + ) def zip_a_file(inpath, outpath): import os import zipfile + with zipfile.ZipFile(outpath, "w", compression=zipfile.ZIP_DEFLATED) as zf: zf.write(inpath, os.path.basename(inpath)) @@ -99,13 +127,33 @@ class Command(BaseCommand): Export all course content to tar.gz and upload it to the course 'GRADES_DOWNLOAD' storage. """ - def add_arguments(self, parser): - parser.add_argument("--username", type=str, help="The username of the user to export the course content") - parser.add_argument("--index", type=int, default=0, help="Start index of the course ids to begin exporting") - parser.add_argument("course_ids", nargs="*", metavar="course_id", default=None, help="Course ids to export or if omitted, all courses will be exported") + """ + Add arguments to the command + """ + parser.add_argument( + "--username", + type=str, + help="The username of the user to export the course content", + ) + parser.add_argument( + "--index", + type=int, + default=0, + help="Start index of the course ids to begin exporting", + ) + parser.add_argument( + "course_ids", + nargs="*", + metavar="course_id", + default=None, + help="Course ids to export or if omitted, all courses will be exported", + ) def log_msg(self, msg): + """ + Log a message and flush it right away. + """ self.stdout.write(msg) self.stdout.flush() @@ -121,66 +169,89 @@ def handle(self, *args, **options): username = options.get("username", None) user = User.objects.get(username=username) - user_id = user.id start_index = options.get("index", None) course_ids.sort() courses_count = len(course_ids) for index in range(start_index, courses_count): course_id = course_ids[index] - self.log_msg(f"Processing {index+1} of {courses_count} - exporting {course_id}") + self.log_msg( + f"Processing {index+1} of {courses_count} - exporting {course_id}" + ) try: course_key = CourseKey.from_string(course_id) cms_root_url = SiteConfiguration.get_value_for_org( course_key.org, "CMS_ROOT_URL", settings.CMS_ROOT_URL ) - cms_export_download_url = ( - f"{cms_root_url}/export/{course_id}" - ) + cms_export_download_url = f"{cms_root_url}/export/{course_id}" lms_root_url = SiteConfiguration.get_value_for_org( course_key.org, "LMS_ROOT_URL", settings.LMS_ROOT_URL ) lms_instructor_data_download_url = ( f"{lms_root_url}/courses/{course_id}/instructor#view-data_download" ) - task_status = _latest_task_status(MockRequest(user=user), str(course_key)) + task_status = _latest_task_status( + MockRequest(user=user), str(course_key) + ) if task_status and task_status.state == UserTaskStatus.SUCCEEDED: artifact = None try: - artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + artifact = UserTaskArtifact.objects.get( + status=task_status, name="Output" + ) data_root = path(settings.GITHUB_REPO_ROOT) - subdir = base64.urlsafe_b64encode(repr(str(course_key)).encode('utf-8')).decode('utf-8') + subdir = base64.urlsafe_b64encode( + repr(str(course_key)).encode("utf-8") + ).decode("utf-8") course_dir = data_root / subdir temp_filepath = course_dir / "export.tar.gz" if not course_dir.isdir(): os.mkdir(course_dir) - with course_import_export_storage.open(artifact.file.name, 'rb') as source: - with open(temp_filepath, 'wb') as destination: + with course_import_export_storage.open( + artifact.file.name, "rb" + ) as source: + with open(temp_filepath, "wb") as destination: + def read_chunk(): """ Read and return a sequence of bytes from the source file. """ return source.read(FILE_READ_CHUNK) - for chunk in iter(read_chunk, b''): + for chunk in iter(read_chunk, b""): destination.write(chunk) finally: if artifact: artifact.file.close() - self.log_msg(f"Download file of the course: {course_id} from: {cms_export_download_url} now uploading to: {lms_instructor_data_download_url}") - upload_tar_gz(temp_filepath, "export_course_content", course_key, artifact.created) + self.log_msg( + f"Download file of the course: {course_id} from: {cms_export_download_url}" + f"now uploading to: {lms_instructor_data_download_url}" + ) + upload_tar_gz( + temp_filepath, + "export_course_content", + course_key, + artifact.created, + ) - self.log_msg(f"Sent export to report store with success of the course: {course_id} from: {cms_export_download_url} to: {lms_instructor_data_download_url}") + self.log_msg( + f"Sent export to report store with success of the course: {course_id} from:" + f"{cms_export_download_url} to: {lms_instructor_data_download_url}" + ) os.remove(temp_filepath) else: - self.log_msg(f"No export found for {course_id}, you can confirm the absent of the export file on: {cms_export_download_url}") + self.log_msg( + f"No export found for {course_id}, you can confirm the absent of the export" + f" file on: {cms_export_download_url}" + ) except Exception as e: self.log_msg(f"Error exporting course {course_id}: {e}") # print stacktrace and continue import traceback + self.log_msg(traceback.format_exc()) continue