Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature-70: Sphinx Autodocs #77

Merged
merged 3 commits into from
Dec 29, 2023
Merged

Feature-70: Sphinx Autodocs #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
baf374a
Refactor 'filehashstore' module to use reStructuredText for sphinx-au…
doulikecookiedough Dec 29, 2023
f5bf47e
Refactor 'client' module to use reStructuredText for sphinx-autodoc
doulikecookiedough Dec 29, 2023
97ea4e8
Refactor 'hashstore' module to use reStructuredText for sphinx-autodoc
doulikecookiedough Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 51 additions & 61 deletions src/hashstore/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,12 @@ def __init__(self):
def load_store_properties(self, hashstore_yaml):
"""Get and return the contents of the current HashStore config file.

Returns:
hashstore_yaml_dict (dict): HashStore properties with the following keys (and values):
store_depth (int): Depth when sharding an object's hex digest.
store_width (int): Width of directories when sharding an object's hex digest.
store_algorithm (str): Hash algorithm used for calculating the object's hex digest.
store_metadata_namespace (str): Namespace for the HashStore's system metadata.
:return: HashStore properties with the following keys (and values):
- store_depth (int): Depth when sharding an object's hex digest.
- store_width (int): Width of directories when sharding an object's hex digest.
- store_algorithm (str): Hash algorithm used for calculating the object's hex digest.
- store_metadata_namespace (str): Namespace for the HashStore's system metadata.
:rtype: dict
"""
property_required_keys = [
"store_depth",
Expand Down Expand Up @@ -234,11 +234,11 @@ class HashStoreClient:
MET_TYPE = "metadata"

def __init__(self, properties, testflag=None):
"""Initialize HashStore and MetacatDB
"""Store objects in a given directory into HashStore.

Args:
properties: See FileHashStore for dictionary example
testflag (str): "knbvm" to initialize MetacatDB
:param str origin_dir: Directory to convert.
:param str obj_type: Type of objects ('object' or 'metadata').
:param int num: Number of files to store.
"""
factory = HashStoreFactory()

Expand All @@ -258,12 +258,11 @@ def __init__(self, properties, testflag=None):
# Methods relating to testing HashStore with knbvm (test.arcticdata.io)

def store_to_hashstore_from_list(self, origin_dir, obj_type, num):
"""Store objects in a given directory into HashStore
"""Store objects in a given directory into HashStore.

Args:
origin_dir (str): Directory to convert
obj_type (str): 'object' or 'metadata'
num (int): Number of files to store
:param str origin_dir: Directory to convert.
:param str obj_type: Type of objects ('object' or 'metadata').
:param int num: Number of files to store.
"""
info_msg = f"HashStore Client - Begin storing {obj_type} objects."
logging.info(info_msg)
Expand Down Expand Up @@ -310,8 +309,7 @@ def store_to_hashstore_from_list(self, origin_dir, obj_type, num):
def try_store_object(self, obj_tuple):
"""Store an object to HashStore and log exceptions as warning.

Args:
obj_tuple: See HashStore store_object signature for details.
:param obj_tuple: See HashStore store_object signature for details.
"""
try:
self.hashstore.store_object(*obj_tuple)
Expand All @@ -321,10 +319,10 @@ def try_store_object(self, obj_tuple):
print(so_exception)

def try_store_metadata(self, obj_tuple):
"""Store an object to HashStore and log exceptions as warning.
"""Store a metadata document to HashStore and log exceptions as warning.

Args:
obj_tuple: See HashStore store_object signature for details.
obj_tuple: See HashStore store_metadata signature for details.
"""
try:
self.hashstore.store_metadata(*obj_tuple)
Expand All @@ -336,10 +334,9 @@ def try_store_metadata(self, obj_tuple):
def retrieve_and_validate_from_hashstore(self, origin_dir, obj_type, num):
"""Retrieve objects or metadata from a Hashstore and validate the content.

Args:
origin_dir (str): Directory to convert
obj_type (str): 'object' or 'metadata'
num (int): Number of files to store
:param str origin_dir: Directory to convert.
:param str obj_type: Type of objects ('object' or 'metadata').
:param int num: Number of files to store.
"""
info_msg = (
f"HashStore Client - Begin retrieving and validating {obj_type} objects."
Expand Down Expand Up @@ -384,8 +381,7 @@ def retrieve_and_validate_from_hashstore(self, origin_dir, obj_type, num):
def validate_object(self, obj_tuple):
"""Retrieves an object from HashStore and validates its checksum.

Args:
obj_tuple: pid_guid, obj_checksum_algo, obj_checksum
:param obj_tuple: Tuple containing pid_guid, obj_checksum_algo, obj_checksum.
"""
pid_guid = obj_tuple[0]
algo = obj_tuple[1]
Expand All @@ -407,10 +403,9 @@ def validate_object(self, obj_tuple):
return

def validate_metadata(self, obj_tuple):
"""Retrieves a metadata from HashStore and validates its checksum
"""Retrieves a metadata from HashStore and validates its checksum.

Args:
obj_tuple: pid_guid, format_id, obj_checksum, obj_algorithm
:param obj_tuple: Tuple containing pid_guid, format_id, obj_checksum, obj_algorithm.
"""
pid_guid = obj_tuple[0]
namespace = obj_tuple[1]
Expand All @@ -433,11 +428,11 @@ def validate_metadata(self, obj_tuple):
return

def delete_objects_from_list(self, origin_dir, obj_type, num):
"""Store objects in a given directory into HashStore
Args:
origin_dir (str): Directory to convert
obj_type (str): 'object' or 'metadata'
num (int): Number of files to store
"""Deletes objects in a given directory into HashStore.

:param str origin_dir: Directory to convert.
:param str obj_type: Type of objects ('object' or 'metadata').
:param int num: Number of files to store.
"""
info_msg = f"HashStore Client - Begin deleting {obj_type} objects."
logging.info(info_msg)
Expand Down Expand Up @@ -482,10 +477,9 @@ def delete_objects_from_list(self, origin_dir, obj_type, num):
logging.info(content)

def try_delete_object(self, obj_pid):
"""Delete an object to HashStore and log exceptions as warning.
"""Delete an object from HashStore and log exceptions as a warning.

Args:
obj_pid (str): Pid of object to delete
:param str obj_pid: PID of the object to delete.
"""
try:
self.hashstore.delete_object(obj_pid)
Expand All @@ -495,10 +489,9 @@ def try_delete_object(self, obj_pid):
print(do_exception)

def try_delete_metadata(self, obj_tuple):
"""Delete an object to HashStore and log exceptions as warning.
"""Delete an object from HashStore and log exceptions as a warning.

Args:
obj_tuple: pid_guid, format_id (namespace)
:param obj_tuple: Tuple containing the PID and format ID (namespace).
"""
pid_guid = obj_tuple[0]
namespace = obj_tuple[1]
Expand Down Expand Up @@ -544,11 +537,10 @@ def __init__(self, hashstore_path, hashstore):
self.db_yaml_dict[key] = checked_property

def get_object_metadata_list(self, origin_directory, num):
"""Query the metacat db for the full obj and metadata list and order by guid.
"""Query the Metacat database for the full object and metadata list, ordered by GUID.

Args:
origin_directory (string): 'var/metacat/data' or 'var/metacat/documents'
num (int): Number of rows to retrieve from metacat db
:param str origin_directory: 'var/metacat/data' or 'var/metacat/documents'.
:param int num: Number of rows to retrieve from the Metacat database.
"""
# Create a connection to the database
db_user = self.db_yaml_dict["db_user"]
Expand Down Expand Up @@ -610,15 +602,14 @@ def get_object_metadata_list(self, origin_directory, num):
def refine_list_for_objects(self, metacat_obj_list, action):
"""Refine a list of objects by checking for file existence and removing duplicates.

Args:
metacat_obj_list (List): List of tuple objects representing rows from metacat db
action (string): "store", "retrieve" or "delete".
"store" will create a list of objects to store that do not exist in HashStore.
"retrieve" will create a list of objects that exist in HashStore.
"delete" will create a list of object pids

Returns:
refined_object_list (List): List of tuple objects based on "action"
:param List metacat_obj_list: List of tuple objects representing rows from Metacat database.
:param str action: Action to perform. Options: "store", "retrieve", or "delete".
- "store": Create a list of objects to store that do not exist in HashStore.
- "retrieve": Create a list of objects that exist in HashStore.
- "delete": Create a list of object PIDs to delete.

:return: Refined list of tuple objects based on the specified action.
:rtype: List
"""
refined_object_list = []
for tuple_item in metacat_obj_list:
Expand Down Expand Up @@ -663,15 +654,14 @@ def refine_list_for_objects(self, metacat_obj_list, action):
def refine_list_for_metadata(self, metacat_obj_list, action):
"""Refine a list of metadata by checking for file existence and removing duplicates.

Args:
metacat_obj_list (List): List of tuple objects representing rows from metacat db
action (string): "store", "retrieve" or "delete".
"store" will create a list of metadata to store that do not exist in HashStore.
"retrieve" will create a list of metadata that exist in HashStore.
"delete" will create a list of metadata pids with their format_ids

Returns:
refined_object_list (List): List of tuple metadata based on "action"
:param List metacat_obj_list: List of tuple objects representing rows from metacat db.
:param str action: Action to perform - "store", "retrieve", or "delete".
- "store": Create a list of metadata to store that do not exist in HashStore.
- "retrieve": Create a list of metadata that exist in HashStore.
- "delete": Create a list of metadata pids with their format_ids.

:return: List of tuple metadata based on the specified action.
:rtype: List
"""
refined_metadata_list = []
for tuple_item in metacat_obj_list:
Expand Down
Loading