Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue#5 #8

Merged
merged 3 commits into from
Aug 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ This is a cli tool to extract table data from a fits file and write it down in a

| Database | Supported |
|----------|:-------------:|
| MySql| YES |
| MySql & MariaDB| YES |
| DuckDB | In progress |
| Postgres | under validation |

Expand Down
2 changes: 2 additions & 0 deletions docs/reference/adapters_base.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# ADAPTERS.BASE Module
::: fits2db.adapters.base
2 changes: 2 additions & 0 deletions docs/reference/adapters_meta.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# ADAPTERS.META Module
::: fits2db.adapters.meta
3 changes: 3 additions & 0 deletions docs/reference/adapters_mysql.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# ADAPTERS.MYSQL Module

::: fits2db.adapters.mysql
3 changes: 3 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# CLI Module
::: fits2db.cli.helper_func
::: fits2db.cli.utils
5 changes: 5 additions & 0 deletions docs/reference/config.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# CONFIG Module

::: fits2db.config.config

::: fits2db.config.config_model
2 changes: 2 additions & 0 deletions docs/reference/core.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Module core
::: fits2db.core
3 changes: 0 additions & 3 deletions docs/reference/database.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,3 @@ When we upload data to the a Database the main idea is to have a mantainable str
`FITS2DB_META` contains the the meta data of each file that is loaded into the database.

::: fits2db.adapters.meta

## Base connection functions
::: fits2db.adapters.base
2 changes: 2 additions & 0 deletions docs/reference/fits.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# FITS Module
::: fits2db.fits
2 changes: 2 additions & 0 deletions docs/reference/log.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# LOG Module
::: fits2db.log
12 changes: 11 additions & 1 deletion fits2db/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,12 @@
"""
Fits2db: A command-line tool to load and manage FITS files in a SQL database.

This tool is designed to be database-agnostic and easily extensible,
leveraging SQLAlchemy for efficient database management.

For detailed documentation, visit: https://pmodwrc.github.io/fits2db/
"""

from .core import Fits2db
__all__ = ["Fits2db"]

__all__ = ["Fits2db"]
59 changes: 39 additions & 20 deletions fits2db/adapters/__init__.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,37 @@
"""
This module provides the DBWriter class, which manages database operations
for FITS files using a configurable loader such as MySQL. It supports operations
like upsert, update, and database cleaning.

Classes:
DBWriter: Handles database operations for FITS files based on the provided configuration.
"""

import logging
from typing import Optional

from pandas import DataFrame

from ..config.config_model import ConfigType
from ..fits import FitsFile
from .mysql import MySQL


# Use the configured log
log = logging.getLogger('fits2db')
log = logging.getLogger("fits2db")


class DBWriter:
def __init__(self, config: ConfigType, file: FitsFile=None):
"""
Handles database operations for FITS files based on the provided configuration.

Attributes:
file (FitsFile): FITS file to be processed.
config (ConfigType): Configuration settings for the database.
db_type (Optional[str]): The type of database (e.g., "mysql").
loader (Optional[MySQL]): The database loader instance.
"""

def __init__(self, config: ConfigType, file: FitsFile = None) -> None:
"""
Initializes the DBWriter class.

Expand All @@ -30,17 +51,16 @@
Returns the database loader based on the configuration.

Returns:
Optional[MySQL]: An instance of the MySQL loader if the database type is MySQL, otherwise None.
Optional[MySQL]: An instance of the MySQL loader if the
database type is MySQL, otherwise None.
"""
log.debug("Getting database loader for type: %s", self.db_type)
if self.db_type and self.db_type.lower() == 'mysql':
if self.db_type and self.db_type.lower() == "mysql":
log.info("MySQL loader created.")
return MySQL(self.config, self.file)

log.debug("No loader created. Database type is not MySQL.")
return None



def _load_db(self) -> Optional[MySQL]:
"""
Expand All @@ -59,15 +79,12 @@
log.warning("Loader initialization failed.")
return loader
except KeyError as e:
log.error(f"Configuration key error: {e}")
log.error("Configuration key error: %s", e)

Check warning on line 82 in fits2db/adapters/__init__.py

View check run for this annotation

Codecov / codecov/patch

fits2db/adapters/__init__.py#L82

Added line #L82 was not covered by tests
return None
except Exception as e:
log.error(f"Unexpected error while loading database: {e}")
return None


def clean_db(self) -> None:
"""
Deletes all fis2db created tables
Deletes all tables created by FITS2DB in the database.
"""
log.debug("Starting db cleaning operation.")
try:
Expand All @@ -79,9 +96,13 @@
except Exception as e:
log.error(f"Error during upsert operation: {e}")

def get_db_file_infos(self)-> DataFrame:
def get_db_file_infos(self) -> Optional[DataFrame]:
"""
Gets all file infos from FITS2DB_META Table
Retrieves file information from the FITS2DB_META table.

Returns:
Optional[DataFrame]: A DataFrame containing file information from
the database, or None if an error occurs.
"""
log.debug("Starting db cleaning operation.")
try:
Expand All @@ -94,8 +115,6 @@
except Exception as e:
log.error(f"Error during upsert operation: {e}")



def upsert(self) -> None:
"""
Inserts or updates data in the database.
Expand All @@ -112,16 +131,16 @@

def update(self) -> None:
"""
Updates data in the database.
Updates data in the database and closes the connection.
"""
log.debug("Starting update operation.")
try:
if self.loader:
self.loader.upsert_file()
self.loader.update_file()

Check warning on line 139 in fits2db/adapters/__init__.py

View check run for this annotation

Codecov / codecov/patch

fits2db/adapters/__init__.py#L139

Added line #L139 was not covered by tests
log.info("Update operation completed successfully.")
self.loader.close_connection()
log.info("Connection closed")
else:
log.error("Loader is not initialized.")
except Exception as e:
log.error(f"Error during update operation: {e}")
log.error(f"Error during update operation: {e}")

Check warning on line 146 in fits2db/adapters/__init__.py

View check run for this annotation

Codecov / codecov/patch

fits2db/adapters/__init__.py#L146

Added line #L146 was not covered by tests
Loading