From d5909213bc53b17b2dc906c9e584a1a2c5544b58 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Tue, 3 Dec 2024 22:18:59 -0500 Subject: [PATCH] chore: Updating Python Requirements --- .pylintrc | 747 ++++++++++++++++++++++++++--------- requirements/base.txt | 4 +- requirements/test.txt | 16 +- scripts/aws/bootstrap.py | 2 +- scripts/aws/common/deploy.py | 1 + scripts/aws/deploy.py | 114 ++++-- scripts/aws/monitor.py | 167 +++++--- 7 files changed, 757 insertions(+), 294 deletions(-) diff --git a/.pylintrc b/.pylintrc index 8e11283..1dbe76f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,274 +1,645 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked and +# will not be imported (useful for modules/projects where namespaces are +# manipulated during runtime and thus existing member attributes cannot be +# deduced by static analysis). It supports qualified module names, as well as +# Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook='' +#init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS, migrations, settings, wsgi.py +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -[MESSAGES CONTROL] +# Pickle collected data for later comparisons. +persistent=yes -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. -#enable= +# Resolve imports to .pyi stubs if available. May reduce no-member messages and +# increase not-an-iterable messages. +prefer-stubs=no -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -disable= -# Never going to use these -# I0011: Locally disabling W0232 -# W0141: Used builtin function 'map' -# W0142: Used * or ** magic -# R0921: Abstract class not referenced -# R0922: Abstract class is only referenced 1 times - I0011,W0141,W0142,R0921,R0922, - -# Django makes classes that trigger these -# W0232: Class has no __init__ method - W0232, - -# Might use these when the code is in better shape -# C0302: Too many lines in module -# R0201: Method could be a function -# R0901: Too many ancestors -# R0902: Too many instance attributes -# R0903: Too few public methods (1/2) -# R0904: Too many public methods -# R0911: Too many return statements -# R0912: Too many branches -# R0913: Too many arguments -# R0914: Too many local variables - C0302,R0201,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914, -# W0511: TODOs etc - W0511, -# E1103: maybe no member - E1103, -# C0111: missing docstring (handled by pep257) - C0111, - - duplicate-code, - -# We can decide if names are invalid on our own - invalid-name, +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.8 -[REPORTS] +# Discover python modules and packages in the file system subtree. +recursive=no -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=text +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes -# Tells whether to display a full report or only the messages -reports=no +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -[TYPECHECK] +[BASIC] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members= - REQUEST, - acl_users, - aq_parent, - objects, - DoesNotExist, - can_read, - can_write, - get_url, - size, - content, - status_code, -# For factory_boy factories - create +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -[BASIC] +# Naming style matching correct class names. +class-naming-style=PascalCase -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$ +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct function names. +function-naming-style=snake_case -# Regular expression which should only match correct method names -method-rgx=([a-z_][a-z0-9_]{2,60}|setUp|set[Uu]pClass|tearDown|tear[Dd]ownClass|assert[A-Z]\w*)$ +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Naming style matching correct method names. +method-naming-style=snake_case -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__|test_.*|setUp|tearDown +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= +# Naming style matching correct module names. +module-naming-style=snake_case -[MISCELLANEOUS] +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + # Maximum number of characters on a single line. max-line-length=120 -# Maximum number of lines in a module +# Maximum number of lines in a module. max-module-lines=1000 -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + too-many-arguments, + use-symbolic-message-instead, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + +# Let 'consider-using-join' be raised when the separator to join on would be +# non-empty (resulting in expected fixes of the type: ``"- " + " - +# ".join(items)``) +suggest-join-with-non-empty-separator=yes + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: text, parseable, colorized, +# json2 (improved json format), json (old json format) and msvs (visual +# studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes [SIMILARITIES] +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + # Minimum lines number of a similarity. min-similarity-lines=4 -# Ignore comments when computing similarities. -ignore-comments=yes -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +[SPELLING] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -[VARIABLES] +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. +spelling-dict= -# Tells whether we should check for unused import in __init__ files. -init-import=no +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_|dummy|unused|.*_unused +# List of comma separated words that should not be checked. +spelling-ignore-words= -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec +[STRING] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +[TYPECHECK] -[DESIGN] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of arguments for function / method -max-args=5 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of locals for function / method body -max-locals=15 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of return / yield for function / method body -max-returns=6 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of branch for function / method body -max-branchs=12 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of statements in function / method body -max-statements=50 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Maximum number of parents for a class (see R0901). -max-parents=7 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# List of decorators that change the signature of a decorated function. +signature-mutators= -[CLASSES] +[VARIABLES] -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes +# List of names allowed to shadow builtins +allowed-redefined-builtins= -[EXCEPTIONS] +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/requirements/base.txt b/requirements/base.txt index 2093e5d..bbf821c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,9 +6,9 @@ # boto==2.49.0 # via google-compute-engine -boto3==1.35.73 +boto3==1.35.74 # via -r requirements/base.in -botocore==1.35.73 +botocore==1.35.74 # via # -r requirements/base.in # boto3 diff --git a/requirements/test.txt b/requirements/test.txt index 39b0fc1..25bb6ad 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,7 +6,7 @@ # annotated-types==0.7.0 # via pydantic -astroid==3.2.4 +astroid==3.3.5 # via pylint attrs==24.2.0 # via @@ -20,12 +20,12 @@ boto==2.49.0 # via # -r requirements/base.txt # google-compute-engine -boto3==1.35.73 +boto3==1.35.74 # via # -r requirements/base.txt # aws-sam-translator # moto -botocore==1.35.73 +botocore==1.35.74 # via # -r requirements/base.txt # aws-xray-sdk @@ -129,14 +129,12 @@ pycparser==2.22 # via cffi pycurl==7.45.3 # via pyresttest -pydantic==2.10.2 +pydantic==2.10.3 # via aws-sam-translator pydantic-core==2.27.1 # via pydantic -pylint==3.2.6 - # via - # -c requirements/constraints.txt - # -r requirements/test.in +pylint==3.3.2 + # via -r requirements/test.in pyparsing==3.2.0 # via moto pyresttest==1.7.1 @@ -173,7 +171,7 @@ responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.22.0 +rpds-py==0.22.1 # via # jsonschema # referencing diff --git a/scripts/aws/bootstrap.py b/scripts/aws/bootstrap.py index a7824ab..f4e2e13 100755 --- a/scripts/aws/bootstrap.py +++ b/scripts/aws/bootstrap.py @@ -119,7 +119,7 @@ def create_base_path_mapping(rest_api_id, api_base, stage_name): def file_arg_to_string(filename): - + """Returns file content as string""" with open(filename, "r", encoding='utf-8') as fin: body = fin.read() diff --git a/scripts/aws/common/deploy.py b/scripts/aws/common/deploy.py index 407fa89..c6ba673 100755 --- a/scripts/aws/common/deploy.py +++ b/scripts/aws/common/deploy.py @@ -103,6 +103,7 @@ def update_stage(client, rest_api_id, stage_name, stage_settings): def deploy(cli_args, integration_settings, stage_settings): + """Deploys an API to a new stage based on provided settings and domain.""" session = botocore.session.get_session() apig = session.create_client('apigateway', cli_args.aws_region) diff --git a/scripts/aws/deploy.py b/scripts/aws/deploy.py index 6b79f59..db28108 100755 --- a/scripts/aws/deploy.py +++ b/scripts/aws/deploy.py @@ -13,44 +13,82 @@ parser = argparse.ArgumentParser() parser.add_argument("--aws-region", required=False, default="us-east-1") - parser.add_argument("--api-base-domain", required=True, - help="The name of the API Gateway domain to be created.") - parser.add_argument("--swagger-filename", required=True, - help="The name of a complete Swagger 2.0 specification file with AWS vendor hooks.") - parser.add_argument("--tag", required=True, - help="Unique identifier for this deployment (such as a git hash)") - parser.add_argument("--rotation-order", required=True, nargs='+', - help="Ordered list of stages in the deployment ring (ex: 'red black')") - parser.add_argument("--log-level", required=False, default="OFF", choices=['OFF', 'ERROR', 'INFO'], - help="Verbosity of messages sent to CloudWatch Logs") - parser.add_argument("--metrics", required=False, default="false", choices=['false', 'true'], - help="Enable CloudWatch metrics") - parser.add_argument("--caching", required=False, default="false", choices=['false', 'true'], - help="Enable API Gateway caching feature") - parser.add_argument("--rate-limit", required=False, default="500", type=str, - help="Default per-resource average rate limit") - parser.add_argument("--burst-limit", required=False, default="1000", type=str, - help="Default per-resource maximum rate limit") - parser.add_argument("--landing-page", required=True, - help="Location of landing page for 'root' level requests") - parser.add_argument("--edxapp-host", required=True, - help="Location of edxapp for request routing") - parser.add_argument("--catalog-host", required=True, - help="Location of catalog IDA for request routing") - parser.add_argument("--enterprise-host", required=False, default='', - help="Location of enterprise IDA for request routing") - parser.add_argument('--analytics-api-host', required=True, - help="Location of analyitcs-api IDA for request routing") - parser.add_argument('--registrar-host', required=True, - help="Location of registrar IDA for request routing") - parser.add_argument('--enterprise-catalog-host', required=True, - help="Location of enterprise catalog IDA for request routing") - parser.add_argument('--authoring-host', required=True, - help="Location of Studio for authoring request routing") - parser.add_argument('--license-manager-host', required=True, - help="Location of License Manager IDA for request routing") - parser.add_argument('--enterprise-access-host', required=True, - help="Location of Enterprise Access IDA for request routing") + parser.add_argument( + "--api-base-domain", required=True, + help="The name of the API Gateway domain to be created." + ) + parser.add_argument( + "--swagger-filename", required=True, + help="The name of a complete Swagger 2.0 specification file with AWS vendor hooks." + ) + parser.add_argument( + "--tag", required=True, + help="Unique identifier for this deployment (such as a git hash)" + ) + parser.add_argument( + "--rotation-order", required=True, nargs='+', + help="Ordered list of stages in the deployment ring (ex: 'red black')" + ) + parser.add_argument( + "--log-level", required=False, default="OFF", choices=['OFF', 'ERROR', 'INFO'], + help="Verbosity of messages sent to CloudWatch Logs" + ) + parser.add_argument( + "--metrics", required=False, default="false", choices=['false', 'true'], + help="Enable CloudWatch metrics" + ) + parser.add_argument( + "--caching", required=False, default="false", choices=['false', 'true'], + help="Enable API Gateway caching feature" + ) + parser.add_argument( + "--rate-limit", required=False, default="500", type=str, + help="Default per-resource average rate limit" + ) + parser.add_argument( + "--burst-limit", required=False, default="1000", type=str, + help="Default per-resource maximum rate limit" + ) + parser.add_argument( + "--landing-page", required=True, + help="Location of landing page for 'root' level requests" + ) + parser.add_argument( + "--edxapp-host", required=True, + help="Location of edxapp for request routing" + ) + parser.add_argument( + "--catalog-host", required=True, + help="Location of catalog IDA for request routing" + ) + parser.add_argument( + "--enterprise-host", required=False, default='', + help="Location of enterprise IDA for request routing" + ) + parser.add_argument( + '--analytics-api-host', required=True, + help="Location of analyitcs-api IDA for request routing" + ) + parser.add_argument( + '--registrar-host', required=True, + help="Location of registrar IDA for request routing" + ) + parser.add_argument( + '--enterprise-catalog-host', required=True, + help="Location of enterprise catalog IDA for request routing" + ) + parser.add_argument( + '--authoring-host', required=True, + help="Location of Studio for authoring request routing" + ) + parser.add_argument( + '--license-manager-host', required=True, + help="Location of License Manager IDA for request routing" + ) + parser.add_argument( + '--enterprise-access-host', required=True, + help="Location of Enterprise Access IDA for request routing" + ) cli_args = parser.parse_args() integration_settings = { diff --git a/scripts/aws/monitor.py b/scripts/aws/monitor.py index bad139b..52c0eea 100755 --- a/scripts/aws/monitor.py +++ b/scripts/aws/monitor.py @@ -21,7 +21,9 @@ # --acct-id 000 --lambda-timeout 10 --lambda-memory 512 --kms-key xxxx-xx-xx-xxx # --subnet-list subnet-112 subnet-113 --sg-list sg-899 sg-901 --environment stage --deployment edx # - +""" +Api Gateway monitoring scripts +""" import logging import time import argparse @@ -49,7 +51,7 @@ def get_api_id(client, api_base_domain): return (response['restApiId'], response['stage']) -def create_api_alarm(cw_session, alarm_name, metric, +def create_api_alarm(*, cw_session, alarm_name, metric, namespace, stat, comparison, description, threshold, period, eval_period, dimensions, topic): """Puts data to the metric, then creates the alarm for appropriate metric in API Gateway""" @@ -92,7 +94,7 @@ def create_lambda_function_zip(jinja_env, temp_dir, splunk_host, splunk_token, l return zip_file -def get_lambda_exec_policy(jinja_env, temp_dir, region, acct_id, func_name, kms_key): +def get_lambda_exec_policy(*, jinja_env, temp_dir, region, acct_id, func_name, kms_key): """updates the policy json and returns it""" resource_values = { 'region': region, @@ -165,7 +167,7 @@ def create_role_with_managed_policy(iam, role_name, assume_role_policy_document, return response['Role']['Arn'] -def create_lambda_function(client, function_name, runtime, role, +def create_lambda_function(*, client, function_name, runtime, role, handler, zip_file, description, timeout, mem_size, vpc): """Creates a lambda function to pull data from cloudwatch event. It only works works in VPC""" @@ -248,20 +250,27 @@ def add_cloudwatchlog_role_to_apigateway(client, role_arn): args = parser.parse_args() session = botocore.session.get_session() - j2_env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), - trim_blocks=False) + j2_env = Environment( + loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=False + ) tmpdirname = tempfile.mkdtemp() - lambda_role_name = args.environment + '-' + args.deployment + '-' + 'lambda-basic-execution-monitor-cloudwatch-logs' + lambda_role_name = ( + args.environment + '-' + args.deployment + '-' + + 'lambda-basic-execution-monitor-cloudwatch-logs' + ) lambda_function_name = args.environment + '-' + args.deployment + '-' + 'cloudwatch-logs-splunk' iam_client = session.create_client('iam', args.aws_region) - cloudwatch_log_role_arn = create_role_with_managed_policy(iam_client, 'apigateway-to-cloudwatch-logs', - '{"Version": "2012-10-17","Statement": ' - '[{"Sid": "","Effect": "Allow","Principal": ' - '{"Service": "apigateway.amazonaws.com"},' - '"Action": "sts:AssumeRole"}]}', - 'arn:aws:iam::aws:policy/service-role/' - 'AmazonAPIGatewayPushToCloudWatchLogs') + cloudwatch_log_role_arn = \ + create_role_with_managed_policy(iam_client, + 'apigateway-to-cloudwatch-logs', + '{"Version": "2012-10-17","Statement": ' + '[{"Sid": "","Effect": "Allow","Principal": ' + '{"Service": "apigateway.amazonaws.com"},' + '"Action": "sts:AssumeRole"}]}', + 'arn:aws:iam::aws:policy/service-role/' + 'AmazonAPIGatewayPushToCloudWatchLogs') logging.info('Waiting for the newly created role to be available') # Sleep for 10 seconds to allow the role created above to be avialable @@ -274,45 +283,81 @@ def add_cloudwatchlog_role_to_apigateway(client, role_arn): sns_client = session.create_client('sns', args.aws_region) cw = session.create_client('cloudwatch', args.aws_region) - create_api_alarm(cw, 'api-gateway-count', 'Count', 'ApiGateway', - 'Average', 'GreaterThanOrEqualToThreshold', - 'Average API count for a period of 5 min', 50, 300, 1, - [{'Name': 'ApiName', 'Value': api_gateway_name}, - {'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}], - get_topic_arn(sns_client, 'aws-non-critical-alert')) - - create_api_alarm(cw, 'api-gateway-latency', 'Latency', 'ApiGateway', 'Average', - 'GreaterThanOrEqualToThreshold', 'Average API Latency for a period of 5 min', 3, 300, 1, - [{'Name': 'ApiName', 'Value': api_gateway_name}, - {'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}], - get_topic_arn(sns_client, 'aws-non-critical-alert')) - - create_api_alarm(cw, 'api-gateway-errors-4xx', '4XXError', 'ApiGateway', 'Average', - 'GreaterThanOrEqualToThreshold', 'Average 4XX errors for a period of 5 min', 4, 300, 1, - [{'Name': 'ApiName', 'Value': api_gateway_name}, - {'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}], - get_topic_arn(sns_client, 'aws-non-critical-alert')) - - create_api_alarm(cw, 'api-gateway-errors-5xx', '5XXError', 'ApiGateway', 'Average', - 'GreaterThanOrEqualToThreshold', 'Average 5XX errors for a period of 5 min', 4, 300, 1, - [{'Name': 'ApiName', 'Value': api_gateway_name}, - {'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}], - get_topic_arn(sns_client, 'aws-non-critical-alert')) - - lambda_exec_role_arn = create_role_with_inline_policy(iam_client, lambda_role_name, - '{"Version": "2012-10-17","Statement": ' - '[{"Effect": "Allow","Principal": ' - '{"Service": "lambda.amazonaws.com"},' - '"Action": "sts:AssumeRole"}]}', - open(get_lambda_exec_policy(j2_env, tmpdirname, # pylint: disable=consider-using-with - args.aws_region, - args.acct_id, - lambda_function_name, - args.kms_key), - encoding='utf-8').read()) + create_api_alarm( + cw_session=cw, alarm_name='api-gateway-count', metric='Count', namespace='ApiGateway', + stat='Average', comparison='GreaterThanOrEqualToThreshold', + description='Average API count for a period of 5 min', + threshold=50, period=300, eval_period=1, + dimensions=[ + {'Name': 'ApiName', 'Value': api_gateway_name}, + {'Name': 'Stage', 'Value': api_stage}, + {'Name': 'ApiId', 'Value': api_id} + ], + topic=get_topic_arn(sns_client, 'aws-non-critical-alert') + ) + + create_api_alarm( + cw_session=cw, alarm_name='api-gateway-latency', metric='Latency', namespace='ApiGateway', + stat='Average', comparison='GreaterThanOrEqualToThreshold', + description='Average API Latency for a period of 5 min', + threshold=3, period=300, eval_period=1, + dimensions=[ + {'Name': 'ApiName', 'Value': api_gateway_name}, + {'Name': 'Stage', 'Value': api_stage}, + {'Name': 'ApiId', 'Value': api_id} + ], + topic=get_topic_arn(sns_client, 'aws-non-critical-alert') + ) + + create_api_alarm( + cw_session=cw, alarm_name='api-gateway-errors-4xx', metric='4XXError', + namespace='ApiGateway', stat='Average', comparison='GreaterThanOrEqualToThreshold', + description='Average 4XX errors for a period of 5 min', + threshold=4, period=300, eval_period=1, + dimensions=[ + {'Name': 'ApiName', 'Value': api_gateway_name}, + {'Name': 'Stage', 'Value': api_stage}, + {'Name': 'ApiId', 'Value': api_id} + ], + topic=get_topic_arn(sns_client, 'aws-non-critical-alert') + ) + + create_api_alarm( + cw_session=cw, alarm_name='api-gateway-errors-5xx', metric='5XXError', + namespace='ApiGateway', stat='Average', comparison='GreaterThanOrEqualToThreshold', + description='Average 5XX errors for a period of 5 min', + threshold=4, period=300, eval_period=1, + dimensions=[ + {'Name': 'ApiName', 'Value': api_gateway_name}, + {'Name': 'Stage', 'Value': api_stage}, + {'Name': 'ApiId', 'Value': api_id} + ], + topic=get_topic_arn(sns_client, 'aws-non-critical-alert') + ) + + with open( + get_lambda_exec_policy( + jinja_env=j2_env, + temp_dir=tmpdirname, + region=args.aws_region, + acct_id=args.acct_id, + func_name=lambda_function_name, + kms_key=args.kms_key + ), + encoding='utf-8' + ) as f: + lambda_exec_role_arn = create_role_with_inline_policy( + iam_client, + lambda_role_name, + '{"Version": "2012-10-17",' + '"Statement": [{"Effect": "Allow","Principal": {"Service": "lambda.amazonaws.com"},' + '"Action": "sts:AssumeRole"}]}', + f.read() + ) logging.info('Waiting for the newly created role to be available') - # Sleep for 10 seconds to allow the role created above to be avialable for lambda function creation + # Sleep for 10 seconds to allow the role created above + # to be available for lambda function creation time.sleep(10) attach_managed_policy(iam_client, lambda_role_name, 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole') @@ -320,11 +365,21 @@ def add_cloudwatchlog_role_to_apigateway(client, role_arn): zip_file_name = create_lambda_function_zip(j2_env, tmpdirname, args.splunk_host, args.splunk_token, lambda_function_name) vpc_config = {'SubnetIds': args.subnet_list, 'SecurityGroupIds': args.sg_list} - create_lambda_function(lambda_client, lambda_function_name, 'nodejs18.x', lambda_exec_role_arn, - 'index.handler', zip_file_name, - 'Demonstrates logging events to Splunk HTTP Event ' - 'Collector, accessing resources in a VPC', args.lambda_timeout, args.lambda_memory, - vpc_config) + create_lambda_function( + client=lambda_client, + function_name=lambda_function_name, + runtime='nodejs18.x', + role=lambda_exec_role_arn, + handler='index.handler', + zip_file=zip_file_name, + description=( + 'Demonstrates logging events to Splunk HTTP Event Collector,' + ' accessing resources in a VPC' + ), + timeout=args.lambda_timeout, + mem_size=args.lambda_memory, + vpc=vpc_config + ) try: shutil.rmtree(tmpdirname) except OSError as exc: