change upsert to build in code and enhance update command

fits2db/adapters/__init__.py

@@ -1,5 +1,6 @@
 import logging
 from typing import Optional
+from pandas import DataFrame
 from ..config.config_model import ConfigType
 from ..fits import FitsFile
 from .mysql import MySQL
@@ -78,6 +79,22 @@ def clean_db(self) -> None:
         except Exception as e:
             log.error(f"Error during upsert operation: {e}")
 
+    def get_db_file_infos(self)-> DataFrame:
+        """
+        Gets all file infos from FITS2DB_META Table
+        """
+        log.debug("Starting db cleaning operation.")
+        try:
+            if self.loader:
+                df = self.loader.get_fits2db_meta()
+                log.info("FITS2DB_META loaded")
+                return df
+            else:
+                log.error("Loader is not initialized.")
+        except Exception as e:
+            log.error(f"Error during upsert operation: {e}")
+
+
 
     def upsert(self) -> None:
         """

fits2db/adapters/base.py

@@ -86,6 +86,15 @@ def clean_db(self):
         log.info(f"Tables in DB:{self.db_table_names}")
         self.drop_tables()
         self.drop_meta_tables()
+
+    def get_fits2db_meta(self)->pd.DataFrame:
+        try:
+            df = pd.read_sql_table("FITS2DB_META", con=self.engine)
+            return df
+        except Exception as err:
+            log.error(err)
+            raise 
+
 
     def upsert_file(self):
         self.session = self.db_session()

fits2db/cli/cli.py

@@ -1,5 +1,5 @@
 import click
-from .helper_func import tables, files, upsert, init
+from .helper_func import tables, files, build, init, update
 from .utils import set_verbosity
 
 @click.version_option("0.0.1b", "--version")
@@ -28,8 +28,9 @@ def cli(ctx):
 
 cli.add_command(files)
 cli.add_command(tables)
-cli.add_command(upsert)
+cli.add_command(build)
 cli.add_command(init)
+cli.add_command(update)
 
 if __name__ == "__main__":
     cli()

fits2db/cli/helper_func.py

@@ -83,6 +83,13 @@ def build(config_path, reset):
     fits = Fits2db(config_path)
     fits.build(reset)
 
+@click.command()
+@click.argument("config_path", default=".", type=click.Path(exists=True))
+def update(config_path):
+    """Upsert all tables defnied in config.yml to databse"""
+    fits = Fits2db(config_path)
+    fits.update_db()
+
 
 @click.command()
 @click.argument("config_path", default=".", type=click.Path(exists=True))

fits2db/core/core.py

@@ -8,6 +8,7 @@
 import pandas as pd
 from ..adapters import DBWriter
 import logging
+from datetime import datetime
 
 
 # Use the configured logger
@@ -69,6 +70,22 @@ def get_file_names(self) -> list:
         log.debug(f"paths {paths}")
         log.info("run function")
         return list(dict.fromkeys(get_all_fits(paths)))
+
+    def get_file_infos(self) -> pd.DataFrame:
+
+        meta = []
+        for path in self.fits_file_paths:
+            path = Path(path)
+            absolute_path = path.resolve()
+            file_meta = {
+                "filename": path.name,
+                "filepath":absolute_path.as_posix(),
+                "last_file_mutation":datetime.fromtimestamp(os.path.getmtime(absolute_path))}
+            meta.append(file_meta)
+        df = pd.DataFrame(meta)
+        log.debug(df)
+        return df
+
 
     def get_table_names(self):
         self.all_table_names = []
@@ -123,6 +140,35 @@ def build(self, reset:bool=True):
             except ValueError as err:
                 log.error(f"\n {err}")
 
+    def update_db(self):
+        file_infos = self.get_file_infos()
+        log.info(file_infos)
+        writer = DBWriter(self.configs)
+        db_file_infos = writer.get_db_file_infos()
+        log.info(db_file_infos)
+        merged_df = pd.merge(file_infos, db_file_infos, on=['filename', 'filepath'], how='left', suffixes=('_file', '_db'))
+
+
+        filtered_df = merged_df[
+            (merged_df['last_file_mutation_file'] > merged_df['last_file_mutation_db']) | 
+            merged_df['last_file_mutation_db'].isna()
+        ]
+
+        result_df = filtered_df[['filename', 'filepath', 'last_file_mutation_file']].rename(
+            columns={'last_file_mutation_file': 'last_file_mutation'}
+        )
+        log.info(result_df)
+        fits_file_paths = result_df["filepath"].to_list()
+        for path in tqdm(fits_file_paths):
+            path = Path(path)
+            try:
+                file = FitsFile(path)
+                writer = DBWriter(self.configs, file)
+                writer.upsert()
+
+            except ValueError as err:
+                log.error(f"\n {err}")
+
 
     def upsert_to_db(self):
         log.debug("Start upsert to db")