diff --git a/Makefile b/Makefile index 70e8671..a1a1d56 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,9 @@ lint: ## Run linters to check code style $(TOX) pycodestyle ./nau_openedx_extensions $(TOX) isort --check-only --diff ./nau_openedx_extensions -isort: ## Fix Python import sort +lint-fix: ## Fix Python import sort $(TOX) isort ./nau_openedx_extensions + $(TOX) autopep8 --in-place --aggressive --aggressive ./nau_openedx_extensions/*.py # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade. diff --git a/nau_openedx_extensions/apps.py b/nau_openedx_extensions/apps.py index af54792..a40afce 100644 --- a/nau_openedx_extensions/apps.py +++ b/nau_openedx_extensions/apps.py @@ -32,7 +32,7 @@ class NauOpenEdxConfig(AppConfig): }, }, "view_context_config": { - "lms.djangoapp": { + "lms.djangoapp": { "course_dashboard": "nau_openedx_extensions.multi_dashboard.context_processor.get_multi_dashboard_context" # lint-amnesty, pylint: disable=line-too-long # noqa }, }, diff --git a/nau_openedx_extensions/studio/contentstore/tasks.py b/nau_openedx_extensions/studio/contentstore/tasks.py index d0b6965..d646594 100644 --- a/nau_openedx_extensions/studio/contentstore/tasks.py +++ b/nau_openedx_extensions/studio/contentstore/tasks.py @@ -21,12 +21,15 @@ User = get_user_model() + class MockRequest(): def __init__(self, user): self.user = user + FILE_READ_CHUNK = 1024 # bytes + def upload_tar_gz_to_report_store(file, name, course_id, timestamp, config_name="GRADES_DOWNLOAD"): """ Upload given file buffer as a tar.gz file using ReportStore. @@ -49,9 +52,12 @@ def upload_tar_gz(file_name, name, course_key, timestamp, config_name="GRADES_DO The ReportStore sometimes fails to upload some files, so we use aws cli as primary upload method. """ bucket = settings.GRADES_DOWNLOAD.get('BUCKET') if bucket: - AWS_ACCESS_KEY_ID = settings.GRADES_DOWNLOAD.get('STORAGE_KWARGS', {}).get('access_key') or settings.AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY = settings.GRADES_DOWNLOAD.get('STORAGE_KWARGS', {}).get('secret_key') or settings.AWS_SECRET_ACCESS_KEY - AWS_S3_ENDPOINT_URL = settings.GRADES_DOWNLOAD.get('STORAGE_KWARGS', {}).get('endpoint_url') or settings.AWS_S3_ENDPOINT_URL + AWS_ACCESS_KEY_ID = settings.GRADES_DOWNLOAD.get( + 'STORAGE_KWARGS', {}).get('access_key') or settings.AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY = settings.GRADES_DOWNLOAD.get( + 'STORAGE_KWARGS', {}).get('secret_key') or settings.AWS_SECRET_ACCESS_KEY + AWS_S3_ENDPOINT_URL = settings.GRADES_DOWNLOAD.get( + 'STORAGE_KWARGS', {}).get('endpoint_url') or settings.AWS_S3_ENDPOINT_URL report_store = ReportStore.from_config(config_name) report_name = "{course_prefix}_{name}_{timestamp_str}.tar.gz".format( @@ -64,10 +70,16 @@ def upload_tar_gz(file_name, name, course_key, timestamp, config_name="GRADES_DO my_env = os.environ.copy() my_env["AWS_ACCESS_KEY_ID"] = AWS_ACCESS_KEY_ID my_env["AWS_SECRET_ACCESS_KEY"] = AWS_SECRET_ACCESS_KEY - returncode = subprocess.call(['aws', f"--endpoint={AWS_S3_ENDPOINT_URL}", 's3', 'cp', str(file_name), f"s3://{bucket}/{path}"], env=my_env) + returncode = subprocess.call(['aws', + f"--endpoint={AWS_S3_ENDPOINT_URL}", + 's3', + 'cp', + str(file_name), + f"s3://{bucket}/{path}"], + env=my_env) if returncode != 0: raise Exception(f"Failed to upload file to S3. Return code: {returncode}") - else: + else: with open(file_name, mode="r", encoding="utf-8") as file: upload_tar_gz_to_report_store(file, name, course_key, timestamp, config_name) @@ -120,13 +132,14 @@ def read_chunk(): if artifact: artifact.file.close() - log.info("Start downloading the file of the course: %s from: %s now uploading to: %s", course_key_string, cms_export_download_url, lms_instructor_data_download_url) + log.info("Start downloading the file of the course: %s from: %s now uploading to: %s", + course_key_string, cms_export_download_url, lms_instructor_data_download_url) upload_tar_gz(temp_filepath, "export_course_content", course_key, artifact.created) - log.info("Sent export to report store with success of the course: %s from: %s to: %s", course_key_string, cms_export_download_url, lms_instructor_data_download_url) + log.info("Sent export to report store with success of the course: %s from: %s to: %s", + course_key_string, cms_export_download_url, lms_instructor_data_download_url) os.remove(temp_filepath) return course_key_string, True else: log.error("No export found for course %s", course_key_string) return course_key_string, False - diff --git a/nau_openedx_extensions/studio/management/commands/export_course_content_async.py b/nau_openedx_extensions/studio/management/commands/export_course_content_async.py index 6cdaa23..8f85a47 100644 --- a/nau_openedx_extensions/studio/management/commands/export_course_content_async.py +++ b/nau_openedx_extensions/studio/management/commands/export_course_content_async.py @@ -14,8 +14,9 @@ Export all courses: python manage.py cms export_course_content_async --username """ -from django.contrib.auth import get_user_model +from cms.djangoapps.contentstore.tasks import export_olx # lint-amnesty, pylint: disable=import-error from django.conf import settings +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from opaque_keys.edx.keys import CourseKey from openedx.core.djangoapps.site_configuration.models import ( # lint-amnesty, pylint: disable=import-error @@ -23,8 +24,6 @@ ) from xmodule.modulestore.django import modulestore -from cms.djangoapps.contentstore.tasks import export_olx # lint-amnesty, pylint: disable=import-error - User = get_user_model() diff --git a/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py b/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py index 93d0a9f..c8503ac 100644 --- a/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py +++ b/nau_openedx_extensions/studio/management/commands/transfer_export_course_content.py @@ -12,25 +12,24 @@ Export all courses: python manage.py cms transfer_export_course_content --username """ -import os -from path import Path as path import base64 -from django.contrib.auth import get_user_model +import os + +from cms.djangoapps.contentstore.storage import course_import_export_storage +from cms.djangoapps.contentstore.views.import_export import _latest_task_status +from common.djangoapps.util.file import course_filename_prefix_generator # lint-amnesty, pylint: disable=import-error from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile from django.core.management.base import BaseCommand +from lms.djangoapps.instructor_task.models import ReportStore # lint-amnesty, pylint: disable=import-error from opaque_keys.edx.keys import CourseKey from openedx.core.djangoapps.site_configuration.models import ( # lint-amnesty, pylint: disable=import-error SiteConfiguration, ) -from xmodule.modulestore.django import modulestore - -from lms.djangoapps.instructor_task.models import ReportStore # lint-amnesty, pylint: disable=import-error -from common.djangoapps.util.file import course_filename_prefix_generator # lint-amnesty, pylint: disable=import-error -from cms.djangoapps.contentstore.views.import_export import _latest_task_status -from cms.djangoapps.contentstore.storage import course_import_export_storage +from path import Path as path from user_tasks.models import UserTaskArtifact, UserTaskStatus - -from django.core.files.base import ContentFile +from xmodule.modulestore.django import modulestore User = get_user_model() @@ -75,7 +74,8 @@ def upload_tar_gz(file_name, name, course_key, timestamp, config_name="GRADES_DO ) path = report_store.path_to(course_key, report_name, '') - import subprocess, os + import os + import subprocess my_env = os.environ.copy() my_env["AWS_ACCESS_KEY_ID"] = AWS_ACCESS_KEY_ID my_env["AWS_SECRET_ACCESS_KEY"] = AWS_SECRET_ACCESS_KEY @@ -88,7 +88,8 @@ def upload_tar_gz(file_name, name, course_key, timestamp, config_name="GRADES_DO def zip_a_file(inpath, outpath): - import os, zipfile + import os + import zipfile with zipfile.ZipFile(outpath, "w", compression=zipfile.ZIP_DEFLATED) as zf: zf.write(inpath, os.path.basename(inpath))