- """,
- unsafe_allow_html=True
-)
+# st.image("image.png", use_container_width=True)
+
+# st.markdown("### 📊 About the System")
+# st.info(
+# """
+# This AI-powered system uses deep learning to analyze chest X-ray images
+# and detect potential signs of Adenocarcinoma Cancer. The model has been
+# trained on thousands of X-ray images to provide accurate predictions.
+# """
+# )
+# st.markdown("### 🎯 How to Use")
+# st.markdown(
+# """
+# 1. Upload a chest X-ray image (PNG, JPG, JPEG)
+# 2. Wait for the image to process
+# 3. Click the 'Analyze Image' button
+# 4. Review the prediction results
+# """
+# )
+# st.markdown("### ⚠️ Important Note")
+# st.warning(
+# """
+# This tool is for screening purposes only and should not be used as a
+# definitive diagnosis. Always consult with healthcare professionals for
+# proper medical advice.
+# """
+# )
+
+# # Header
+# st.markdown(
+# """
+#
+#
+#
+# AI-Powered Chest Cancer Detection
+#
+#
+# Advanced deep learning system for early detection of Adenocarcinoma Cancer through X-ray analysis
+#
",
# unsafe_allow_html=True
# )
-
-
-
-# ----------------------------
-
-
-# from src.chest_cancer_classifier import logger
-# from chest_cancer_classifier.pipeline.stage_1_data_ingestion import DataIngestionTrainingPipeline
-# from chest_cancer_classifier.pipeline.stage_2_prepare_base_model import PrepareBaseModelTrainingPipeline
-# from chest_cancer_classifier.pipeline.stage_3_model_training import ModelTrainingPipeline
-# from chest_cancer_classifier.pipeline.stage_4_model_evaluation import EvaluationPipeline
-
-# # Define the name of the current stage in the data processing pipeline
-# STAGE_NAME = "Data Ingestion stage"
-
-# try:
-# # Log the start of the data ingestion stage
-# logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
-# # Create an instance of the DataIngestionTrainingPipeline
-# obj = DataIngestionTrainingPipeline()
-# # Execute the main process of the data ingestion pipeline
-# obj.main()
-# # Log the successful completion of the data ingestion stage
-# logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
-# except Exception as e:
-# # Log any exceptions that occur during the execution
-# logger.exception(e)
-# raise e # Reraise the exception for further handling if necessary
-
-
-# # Define the name of the current stage in the data processing pipeline
-# STAGE_NAME = "Prepare Base Model stage"
-
-# try:
-# # Log the start of the stage
-# logger.info(f"*******************")
-# logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
-
-# # Create an instance of the training pipeline and run the main method
-# prepare_base_model = PrepareBaseModelTrainingPipeline()
-# prepare_base_model.main()
-
-# # Log the completion of the stage
-# logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
-# except Exception as e:
-# # Log any exceptions that occur during the execution
-# logger.exception(e)
-# raise e # Reraise the exception for further handling
-
-
-# # Define the name of the current stage in the data processing pipeline
-# STAGE_NAME = "Model Training stage"
-
-# try:
-# # Log the start of the stage
-# logger.info(f"*******************")
-# logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
-
-# # Create an instance of the training pipeline and run the main method
-# model_trainer = ModelTrainingPipeline()
-# model_trainer.main()
-
-# # Log the completion of the stage
-# logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
-# except Exception as e:
-# # Log any exceptions that occur during the execution
-# logger.exception(e)
-# raise e # Reraise the exception for further handling
-
-
-# # Define the name of the current stage in the data processing pipeline
-# STAGE_NAME = "Model Evaluation stage"
-
-# try:
-# # Log the start of the evaluation stage
-# logger.info(f"*******************")
-# logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
-
-# # Create an instance of the EvaluationPipeline class and run the main method
-# model_evaluater = EvaluationPipeline()
-# model_evaluater.main()
-
-# # Log the completion of the evaluation stage
-# logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
-# except Exception as e:
-# # Log any exceptions that occur during the execution
-# logger.exception(e)
-# # Raise the exception to propagate it further
-# raise e
\ No newline at end of file
diff --git a/dvc.lock b/dvc.lock
index 4e025ff..7872204 100644
--- a/dvc.lock
+++ b/dvc.lock
@@ -5,32 +5,32 @@ stages:
deps:
- path: config/config.yaml
hash: md5
- md5: 260e6cd7e270c8e67181a97209d32395
- size: 1280
+ md5: c34b1a3d1078088dcb9f483aa5971ec5
+ size: 1443
- path: src/chest_cancer_classifier/pipeline/stage_1_data_ingestion.py
hash: md5
md5: c86ebcac1360545068f5aadbaf4f776e
size: 1882
outs:
- - path: artifacts/data_ingestion/Chest-CT-Scan-data
+ - path: artifacts/data_ingestion/Data
hash: md5
- md5: 904fa45d934ce879b3b1933dca6cb2f1.dir
- size: 49247431
- nfiles: 343
+ md5: 3ddc2c0e262e947578d73eeec3119af6.dir
+ size: 124956798
+ nfiles: 1000
prepare_base_model:
cmd: python src/chest_cancer_classifier/pipeline/stage_2_prepare_base_model.py
deps:
- path: config/config.yaml
hash: md5
- md5: 260e6cd7e270c8e67181a97209d32395
- size: 1280
+ md5: c34b1a3d1078088dcb9f483aa5971ec5
+ size: 1443
- path: src/chest_cancer_classifier/pipeline/stage_2_prepare_base_model.py
hash: md5
md5: 48cb79ddc79aa7ce7574ea6d66832c48
size: 1935
params:
params.yaml:
- CLASSES: 2
+ CLASSES: 4
IMAGE_SIZE:
- 224
- 224
@@ -41,26 +41,26 @@ stages:
outs:
- path: artifacts/prepare_base_model
hash: md5
- md5: 5f2673ba957c09b60880b6e4e9b9f031.dir
- size: 118073464
+ md5: 093d170faa2be4dcd1f5b855e63f125d.dir
+ size: 118274168
nfiles: 2
training:
cmd: python src/chest_cancer_classifier/pipeline/stage_3_model_training.py
deps:
- - path: artifacts/data_ingestion/Chest-CT-Scan-data
+ - path: artifacts/data_ingestion/Data
hash: md5
- md5: 904fa45d934ce879b3b1933dca6cb2f1.dir
- size: 49247431
- nfiles: 343
+ md5: 3ddc2c0e262e947578d73eeec3119af6.dir
+ size: 124956798
+ nfiles: 1000
- path: artifacts/prepare_base_model
hash: md5
- md5: 5f2673ba957c09b60880b6e4e9b9f031.dir
- size: 118073464
+ md5: 093d170faa2be4dcd1f5b855e63f125d.dir
+ size: 118274168
nfiles: 2
- path: config/config.yaml
hash: md5
- md5: 260e6cd7e270c8e67181a97209d32395
- size: 1280
+ md5: c34b1a3d1078088dcb9f483aa5971ec5
+ size: 1443
- path: src/chest_cancer_classifier/pipeline/stage_3_model_training.py
hash: md5
md5: 7affe72d4439d7780014e9844c727727
@@ -77,24 +77,24 @@ stages:
outs:
- path: artifacts/training/model.h5
hash: md5
- md5: 2f95c5802cf0a0f0bb0e49e98afd1340
- size: 59551208
+ md5: 39925b7fd1a4a6ec85782fb9545927f3
+ size: 60153320
evaluation:
cmd: python src/chest_cancer_classifier/pipeline/stage_4_model_evaluation.py
deps:
- - path: artifacts/data_ingestion/Chest-CT-Scan-data
+ - path: artifacts/data_ingestion/Data
hash: md5
- md5: 904fa45d934ce879b3b1933dca6cb2f1.dir
- size: 49247431
- nfiles: 343
+ md5: 3ddc2c0e262e947578d73eeec3119af6.dir
+ size: 124956798
+ nfiles: 1000
- path: artifacts/training/model.h5
hash: md5
- md5: 2f95c5802cf0a0f0bb0e49e98afd1340
- size: 59551208
+ md5: 39925b7fd1a4a6ec85782fb9545927f3
+ size: 60153320
- path: config/config.yaml
hash: md5
- md5: 260e6cd7e270c8e67181a97209d32395
- size: 1280
+ md5: c34b1a3d1078088dcb9f483aa5971ec5
+ size: 1443
- path: src/chest_cancer_classifier/pipeline/stage_4_model_evaluation.py
hash: md5
md5: 3d60303f5fed6f98b3b16b3fcd8b33fd
@@ -109,5 +109,5 @@ stages:
outs:
- path: scores.json
hash: md5
- md5: 585bcc7607c81863066757a101118260
- size: 69
+ md5: 3e53d3f918e0950a632a92db7ea4eb02
+ size: 73
diff --git a/dvc.yaml b/dvc.yaml
index bb912e1..69fde3c 100644
--- a/dvc.yaml
+++ b/dvc.yaml
@@ -11,7 +11,7 @@ stages:
- config/config.yaml # Configuration file
# Output generated by this stage
outs:
- - artifacts/data_ingestion/Chest-CT-Scan-data # Directory for the ingested data
+ - artifacts/data_ingestion/Data # Directory for the ingested data
# Prepare Base Model Stage
@@ -42,7 +42,7 @@ stages:
deps:
- src/chest_cancer_classifier/pipeline/stage_3_model_training.py # Script file for training the model
- config/config.yaml # Configuration file
- - artifacts/data_ingestion/Chest-CT-Scan-data # Data from the ingestion stage
+ - artifacts/data_ingestion/Data # Data from the ingestion stage
- artifacts/prepare_base_model # Model from the preparation stage
# Parameters used in this stage
params:
@@ -63,7 +63,7 @@ stages:
deps:
- src/chest_cancer_classifier/pipeline/stage_4_model_evaluation.py # Script file for evaluating the model
- config/config.yaml # Configuration file
- - artifacts/data_ingestion/Chest-CT-Scan-data # Data from the ingestion stage
+ - artifacts/data_ingestion/Data # Data from the ingestion stage
- artifacts/training/model.h5 # Model from the training stage
# Parameters used in this stage
params:
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..111bf7e
--- /dev/null
+++ b/main.py
@@ -0,0 +1,86 @@
+
+# complete pipeline
+
+from src.chest_cancer_classifier import logger
+from chest_cancer_classifier.pipeline.stage_1_data_ingestion import DataIngestionTrainingPipeline
+from chest_cancer_classifier.pipeline.stage_2_prepare_base_model import PrepareBaseModelTrainingPipeline
+from chest_cancer_classifier.pipeline.stage_3_model_training import ModelTrainingPipeline
+from chest_cancer_classifier.pipeline.stage_4_model_evaluation import EvaluationPipeline
+
+# Define the name of the current stage in the data processing pipeline
+STAGE_NAME = "Data Ingestion stage"
+
+try:
+ # Log the start of the data ingestion stage
+ logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+ # Create an instance of the DataIngestionTrainingPipeline
+ obj = DataIngestionTrainingPipeline()
+ # Execute the main process of the data ingestion pipeline
+ obj.main()
+ # Log the successful completion of the data ingestion stage
+ logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+ # Log any exceptions that occur during the execution
+ logger.exception(e)
+ raise e # Reraise the exception for further handling if necessary
+
+
+# Define the name of the current stage in the data processing pipeline
+STAGE_NAME = "Prepare Base Model stage"
+
+try:
+ # Log the start of the stage
+ logger.info(f"*******************")
+ logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+
+ # Create an instance of the training pipeline and run the main method
+ prepare_base_model = PrepareBaseModelTrainingPipeline()
+ prepare_base_model.main()
+
+ # Log the completion of the stage
+ logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+ # Log any exceptions that occur during the execution
+ logger.exception(e)
+ raise e # Reraise the exception for further handling
+
+
+# Define the name of the current stage in the data processing pipeline
+STAGE_NAME = "Model Training stage"
+
+try:
+ # Log the start of the stage
+ logger.info(f"*******************")
+ logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+
+ # Create an instance of the training pipeline and run the main method
+ model_trainer = ModelTrainingPipeline()
+ model_trainer.main()
+
+ # Log the completion of the stage
+ logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+ # Log any exceptions that occur during the execution
+ logger.exception(e)
+ raise e # Reraise the exception for further handling
+
+
+# Define the name of the current stage in the data processing pipeline
+STAGE_NAME = "Model Evaluation stage"
+
+try:
+ # Log the start of the evaluation stage
+ logger.info(f"*******************")
+ logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+
+ # Create an instance of the EvaluationPipeline class and run the main method
+ model_evaluater = EvaluationPipeline()
+ model_evaluater.main()
+
+ # Log the completion of the evaluation stage
+ logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+ # Log any exceptions that occur during the execution
+ logger.exception(e)
+ # Raise the exception to propagate it further
+ raise e
\ No newline at end of file
diff --git a/params.yaml b/params.yaml
index a729f64..3522c66 100644
--- a/params.yaml
+++ b/params.yaml
@@ -15,7 +15,8 @@ INCLUDE_TOP: False # Useful when using transfer learning without the top layer
EPOCHS: 5
# Number of classes in the classification task
-CLASSES: 2 # Example: binary classification (e.g., 0 and 1)
+# CLASSES: 2 # Example: binary classification (e.g., 0 and 1)
+CLASSES: 4 # Example: MULTI-CLASS classification (e.g., 0,1,2,3)
# Pre-trained weights to use for the model
WEIGHTS: imagenet # This indicates using ImageNet weights for transfer learning
diff --git a/project_step_by_step_workflow.md b/project_step_by_step_workflow.md
new file mode 100644
index 0000000..809ed3c
--- /dev/null
+++ b/project_step_by_step_workflow.md
@@ -0,0 +1,550 @@
+
+
+------
+
+# **`Step By Step Project Workflow`**
+
+1. **Create** a Github **Repository**.
+ - Add **licence**, **README.md** and **.gitignore** file.
+2. **Clone** the Repository in **Your system**.
+
+3. It is a good approach to create a **Project Template**. It help to **reduce effort** to **create files and folders manually**.
+ - Create a **template.py file**. Add code in that file. Here is the code for this project's **template.py file**.
+
+ ```bash
+ # Import necessory libraries for setting up project template
+
+ import os
+ import logging
+ from pathlib import Path
+
+ # Set up information level log for tracking progress
+ logging.basicConfig(level=logging.INFO, format='[%(asctime)s]: %(message)s:')
+
+ # Set up Project name
+ project_name = "chest_cancer_classifier"
+
+ # Files that i need for this project
+ list_of_files = [
+ ".github/workflows/.gitkeep",
+ f"src/{project_name}/__init__.py",
+ f"src/{project_name}/components/__init__.py",
+ f"src/{project_name}/utils/__init__.py",
+ f"src/{project_name}/config/__init__.py",
+ f"src/{project_name}/config/configuration.py",
+ f"src/{project_name}/pipeline/__init__.py",
+ f"src/{project_name}/entity/__init__.py",
+ f"src/{project_name}/constants/__init__.py",
+ "config/config.yaml",
+ "dvc.yaml",
+ "params.yaml",
+ "requirements.txt",
+ "setup.py",
+ "research/experiments.ipynb",
+ "templates/index.html",
+ "static/style.css",
+ "app.py"
+ ]
+
+
+ # Iterate over each file path in the list of files
+ for filepath in list_of_files:
+ # Convert the filepath string to a Path object for easier manipulation
+ filepath = Path(filepath)
+
+ # Split the filepath into its directory and filename components
+ filedir, filename = os.path.split(filepath)
+
+ # Check if the directory part of the filepath is not empty
+ if filedir != "":
+ # Create the directory if it doesn't exist, without raising an error if it does
+ os.makedirs(filedir, exist_ok=True)
+ logging.info(f"Creating directory; {filedir} for the file: {filename}")
+
+ # Check if the file does not exist or is empty
+ if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
+ # Open the file in write mode (this creates the file if it doesn't exist)
+ with open(filepath, "w") as f:
+ pass # No content is written to the file
+ logging.info(f"Creating empty file: {filepath}")
+
+ else:
+ # Log that the file already exists
+ logging.info(f"{filename} already exists.")
+ ```
+
+ - Now after writing your code it's time to implement this.
+ - Go to your VS Code **terminal** and run this file with this command.
+ ```bash
+ python template.py
+ ```
+ - You will see the whole buch of **files and folders** will be created **automatically**.
+
+4. Next step will be **setting up Project** and **Virtual Environment**.
+
+ - Let's First create a **Virtual Environment**. In my case I am using **conda**.
+ - First command will be to **create** a virtual environment with **specific python version** and **-y flag mean yes to all comming steps**.
+
+ ```bash
+ conda create -p venv python=3.10 -y
+ ```
+ - Now we need to activate the **Created Virtual Environment** with this command.
+ ```bash
+ conda activate venv
+ ```
+ - Now we need to **install specific libraries in this environnment** according to our specific **use case**. **You can install it 1 by 1**.
+ - But more convinent approach wil be to create a **requirements.txt**. Add names of libraries in that file. At the end add `-e .` for project setup.
+ - Next you need to create a **setup.py** file for project setup. This will **contain information related to your project** (author name, project name, libraries used in this project). Here is a the code of this project's **setup.py**.
+
+ ```bash
+ # Import the setuptools library to facilitate packaging and distribution
+ import setuptools
+
+ # Open the README file to read its content for the long description, using UTF-8 encoding
+ with open("README.md", "r", encoding="utf-8") as f:
+ long_description = f.read() # Store the README content
+
+ # Open the requirements.txt file to read all dependencies
+ with open("requirements.txt") as f:
+ install_requires = f.read().splitlines() # Split lines to get each dependency as a list item
+
+ # Define the version of the package
+ __version__ = "0.0.0"
+
+ # Define metadata for the package
+ REPO_NAME = "Chest-Cancer-Classification-Using-MLflow-and-DVC" # GitHub repository name
+ AUTHOR_USER_NAME = "muhammadadilnaeem" # GitHub username
+ SRC_REPO = "chest_cancer_classifier" # Name of the source directory/package
+ AUTHOR_EMAIL = "madilnaeem0@gmail.com" # Contact email for the author
+
+ # Call the setup function to configure the package
+ setuptools.setup(
+ name=SRC_REPO, # Name of the package, must match the directory in 'src'
+ version=__version__, # Version of the package
+ author=AUTHOR_USER_NAME, # Author name as it will appear in package metadata
+ author_email=AUTHOR_EMAIL, # Author's contact email
+ description="A demo python package for Chest Cancer Classification Web Application.", # Short package description
+ long_description=long_description, # Detailed description read from README.md
+ long_description_content_type="text/markdown", # Format of the long description (markdown)
+ url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}", # URL of the package repository
+ project_urls={ # Additional URLs related to the project, such as an issue tracker
+ "Bug Tracker": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}/issues",
+ },
+ package_dir={"": "src"}, # Specify that the root of packages is the 'src' directory
+ packages=setuptools.find_packages(where="src"), # Automatically find all packages in 'src'
+ install_requires=install_requires, # List of dependencies read from requirements.txt
+ classifiers=[ # Additional metadata classifiers that help others find the package
+ "Programming Language :: Python :: 3", # Indicate that the package is written in Python 3
+ "License :: OSI Approved :: MIT License", # Specify the license type
+ "Operating System :: OS Independent", # OS compatibility
+ ],
+ python_requires='>=3.10', # Specify the minimum Python version required
+ )
+ ```
+
+ - But i will install libraries using **requirements.txt** with this command. This will install all mentioned libraries in **requirements.txt** and set up project.
+
+ ```bash
+ pip install -r requirements.txt
+ ```
+
+ - This is an `optional step`. If you want to check what **specific versions of libraries are installed in Your Virtual Environment** use this command.
+ ```bash
+ pip freeze > requirements.txt
+ ```
+
+
+5. Next step would be to set up **logging and exception** for better code readibilty and pracrice.
+ - We need to **set up loging**.
+ - Inside `src` folder we have `__init__.py`. We will write logging code in this file.It will help us direcly import `logger`. Here is what logging code will look like
+ ```bash
+ import os # Import the os module for interacting with the operating system
+ import sys # Import the sys module for system-specific parameters and functions
+ import logging # Import the logging module for logging messages
+
+ # Define the logging format string
+ logging_str = "[%(asctime)s: %(levelname)s: %(module)s: %(message)s]"
+
+ # Specify the directory where log files will be stored
+ log_dir = "logs"
+
+ # Create the full path for the log file
+ log_filepath = os.path.join(log_dir, "running_logs.log")
+
+ # Create the log directory if it does not exist
+ os.makedirs(log_dir, exist_ok=True)
+
+ # Configure the logging settings
+ logging.basicConfig(
+ level=logging.INFO, # Set the logging level to INFO
+ format=logging_str, # Use the defined format for log messages
+ handlers=[
+ logging.FileHandler(log_filepath), # Log messages to a file
+ logging.StreamHandler(sys.stdout) # Also output log messages to the console
+ ]
+ )
+
+ # Create a logger object with a specific name
+ logger = logging.getLogger("chest_cancer_classifier_logger") # This logger can be used throughout the application
+ ```
+
+ - We need to **set up exceptions**. For this project We will set this up using a python package `python-box`. All exceptions code can be found in `common_functions.py` which we will discuss in next step. But first we will use some new packages for this prject and we need to take a look:
+
+ ### **python-box**
+
+ - **Explanation**:
+ `python-box` is a versatile library that extends Python dictionaries, allowing you to access dictionary keys as object attributes (dot-notation). It also includes advanced functionality, such as working seamlessly with JSON, YAML, and other data structures.
+
+ - **Use Case**:
+ - Ideal for managing configuration data or JSON-like structures in an easy-to-access and user-friendly manner.
+
+ - **Example**:
+ ```python
+ from box import Box
+
+ # Original dictionary
+ dic = {"name": "John", "age": 25}
+
+ # Convert to a Box object
+ dic = Box(dic)
+
+ # Access values using dot notation
+ print(dic.name) # Output: John
+ print(dic.age) # Output: 25
+
+ # Adding new key-value pairs
+ dic.country = "USA"
+ print(dic.country) # Output: USA
+ ```
+
+ - **Additional Features**:
+ - Handles nested dictionaries for easy access.
+ - Converts JSON/YAML files directly into Box objects for seamless integration with configuration files.
+
+
+ ### **config-box**
+ - **Explanation**:
+ - `config-box` provides a way to handle Python dictionaries as objects, enabling dot-notation access (e.g., `obj.key`) instead of traditional dictionary indexing (`obj["key"]`). It is especially helpful when working with configuration files like YAML or JSON.
+
+ - **Use Case**:
+ - Simplifies working with nested dictionary configurations.
+
+ - **Example**:
+ ```python
+ from box import ConfigBox
+
+ # Original dictionary
+ dic = {"key": "value", "tala": "chabi"}
+
+ # Convert to ConfigBox
+ dic = ConfigBox(dic)
+
+ # Access value with dot notation
+ print(dic.tala) # Output: chabi
+ ```
+
+ ### **ensure**
+ - **Explanation**:
+ - `ensure` enforces type annotations at runtime. If the provided arguments don't match the annotated types, it raises an error. This is particularly useful for maintaining strict type-checking in functions.
+
+ - **Use Case**:
+ - Ensures functions only receive arguments of expected types.
+
+ - **Example**:
+ ```python
+ from ensure import ensure_annotations
+
+ @ensure_annotations
+ def get_multiplication(x: int, y: int):
+ return x * y
+
+ # Correct usage
+ print(get_multiplication(x=6, y=9)) # Output: 54
+
+ # Incorrect usage
+ print(get_multiplication(x=6, y="9")) # Raises TypeError: argument 'y' must be int
+ ```
+
+
+
+
+
+6. Now it's time to create **functions** that w**e will use again and again in our project**. Instead of writing code again and again for different functions we will just write code and keep it in `common_functions.py`, whenever we need any functionality we will just import that code from this file.
+ - For this I will go to `src` and then `utils` folder.
+ - Inside this folder we will create a file `common_functions.py`, which will look like this
+ ```bash
+ # Import libraries
+ import os
+ import json
+ import yaml
+ import joblib
+ import base64
+ from typing import Any
+ from pathlib import Path
+ from box import ConfigBox
+ from ensure import ensure_annotations
+ from box.exceptions import BoxValueError
+ from chest_cancer_classifier import logger
+
+ @ensure_annotations
+ def read_yaml(path_to_yaml: Path) -> ConfigBox:
+ """
+ Reads a YAML file and returns its contents as a ConfigBox object.
+
+ Args:
+ path_to_yaml (Path): Path to the YAML file.
+
+ Raises:
+ ValueError: If the YAML file is empty.
+ Exception: If any other error occurs during file reading.
+
+ Returns:
+ ConfigBox: Parsed content from YAML as a ConfigBox object.
+ """
+ try:
+ with open(path_to_yaml) as yaml_file:
+ content = yaml.safe_load(yaml_file)
+ logger.info(f"YAML file '{path_to_yaml}' loaded successfully")
+ return ConfigBox(content)
+ except BoxValueError:
+ logger.error(f"YAML file '{path_to_yaml}' is empty")
+ raise ValueError("YAML file is empty")
+ except Exception as e:
+ logger.exception(f"Error loading YAML file '{path_to_yaml}': {e}")
+ raise e
+
+
+ @ensure_annotations
+ def create_directories(path_to_directories: list, verbose=True):
+ """
+ Creates a list of directories if they do not already exist.
+
+ Args:
+ path_to_directories (list): List of directory paths to create.
+ verbose (bool, optional): If True, logs creation for each directory. Defaults to True.
+ """
+ for path in path_to_directories:
+ os.makedirs(path, exist_ok=True)
+ if verbose:
+ logger.info(f"Directory created at: {path}")
+
+
+ @ensure_annotations
+ def save_json(path: Path, data: dict):
+ """
+ Saves a dictionary as a JSON file.
+
+ Args:
+ path (Path): Path where JSON file will be saved.
+ data (dict): Data to save in JSON format.
+ """
+ with open(path, "w") as f:
+ json.dump(data, f, indent=4)
+ logger.info(f"JSON file saved at: {path}")
+
+
+ @ensure_annotations
+ def load_json(path: Path) -> ConfigBox:
+ """
+ Loads JSON data from a file and returns it as a ConfigBox object.
+
+ Args:
+ path (Path): Path to the JSON file.
+
+ Returns:
+ ConfigBox: Data loaded from JSON file as a ConfigBox.
+ """
+ with open(path) as f:
+ content = json.load(f)
+ logger.info(f"JSON file loaded successfully from: {path}")
+ return ConfigBox(content)
+
+
+ @ensure_annotations
+ def save_bin(data: Any, path: Path):
+ """
+ Saves data in binary format using joblib.
+
+ Args:
+ data (Any): Data to save as binary.
+ path (Path): Path where binary file will be saved.
+ """
+ joblib.dump(value=data, filename=path)
+ logger.info(f"Binary file saved at: {path}")
+
+
+ @ensure_annotations
+ def load_bin(path: Path) -> Any:
+ """
+ Loads binary data using joblib.
+
+ Args:
+ path (Path): Path to the binary file.
+
+ Returns:
+ Any: Object stored in the binary file.
+ """
+ data = joblib.load(path)
+ logger.info(f"Binary file loaded from: {path}")
+ return data
+
+
+ @ensure_annotations
+ def get_size(path: Path) -> str:
+ """
+ Gets the file size in kilobytes (KB).
+
+ Args:
+ path (Path): Path of the file.
+
+ Returns:
+ str: Size in KB, rounded to the nearest integer.
+ """
+ size_in_kb = round(os.path.getsize(path) / 1024)
+ logger.info(f"Size of '{path}': ~{size_in_kb} KB")
+ return f"~ {size_in_kb} KB"
+
+
+ def decodeImage(imgstring: str, fileName: str):
+ """
+ Decodes a base64 string and saves it as an image file.
+
+ Args:
+ imgstring (str): Base64-encoded image string.
+ fileName (str): Path where the decoded image will be saved.
+ """
+ imgdata = base64.b64decode(imgstring)
+ with open(fileName, 'wb') as f:
+ f.write(imgdata)
+ logger.info(f"Image decoded and saved to '{fileName}'")
+
+
+ def encodeImageIntoBase64(croppedImagePath: str) -> str:
+ """
+ Encodes an image file as a base64 string.
+
+ Args:
+ croppedImagePath (str): Path to the image file.
+
+ Returns:
+ str: Base64-encoded string of the image file.
+ """
+ with open(croppedImagePath, "rb") as f:
+ encoded_string = base64.b64encode(f.read())
+ logger.info(f"Image at '{croppedImagePath}' encoded to base64")
+ return encoded_string.decode('utf-8')
+ ```
+
+7. Now we need to understand **data Acquisition** for this. You can get data from any where like from databases (mongodb,sql), platforms like (kaggle, uci machine learning).
+ - But for this project we will acquire data from `Google Drive`.
+
+8. Now we have **set up project**, We need to **understand how to write our code step by step**.
+
+ ### **Project Workflow**
+
+ 0. **Perform Experiments in `research` folder**:
+
+ - For every stage we will create seperate **Jupyter notebook**, we will write code in **Jupyter notebook (.ipynb)** for **experiments** and use same code in **python files (.py)**. Here are the names of these files:
+
+ - `01_data_ingestion.ipynb`
+ - `02_prepare_base_model.ipynb`
+ - `03_model_trainer.ipynb`
+ - `04_model_evaluation.ipynb`
+
+ 1. **Update `config.yaml`**:
+ Define project-wide static configurations, such as file paths, thresholds, or model parameters.
+
+ 2. **Update `secrets.yaml` (Optional)**:
+ Store sensitive data like API keys or passwords securely (if required).
+
+ 3. **Update `params.yaml`**:
+ Define tunable parameters for experiments, such as learning rates or batch sizes.
+ - We will load parameters of `VGG16`
+
+ 4. **Update the Entity**:
+ Create structured classes to represent data models, ensuring type safety and clarity.
+ - In our case we have file `config_entity.py`. This file has all entities used in this project for configuration.
+
+ 5. **Update the Configuration Manager**:
+ Implement a utility in `src/config` to parse and manage configuration files dynamically.
+ - In our case we have `configuration.py`.
+
+ 6. **Update the Components**:
+ Write modular functions or classes in `src/components` to perform specific tasks like data preprocessing or model training. Here are the folloowing components used in this project:
+ - `data_ingestion.py`.
+ - `prepare_base_models.py`.
+ - `model_trainer.py`.
+ - `model_evaluation_with_mlflow.py`.
+
+ 7. **Update the Pipeline**:
+ Integrate components in `src/pipeline` to define end-to-end workflows (e.g., data processing to model inference).
+ - Our pipeline stages step by step will be
+
+ - `stage_1_data_ingestion.py`
+ - `stage_2_prepare_base_model.py`
+ - `stage_3_model_training.py`
+ - `stage_4_model_evaluation.py`
+ - `stage_5_prediction.py`
+
+ 8. **Update the `main.py`**:
+ Write the entry point script to execute the pipeline and orchestrate the workflow.
+
+ 9. **Update the `dvc.yaml`**:
+ Define and version your data pipeline with DVC, including stages for data preparation, model training, and evaluation.
+
+9. We can **track experiments** that we perform with our code using **Open Source** `MLops Tools` like **Dagshub**, **MLflow** and **DVC**.
+
+ - For this project we will first set up our Dagshub account and then use our **github repository** in **Dagshub** for **experiment tracking using MLflow**.
+
+ - An example of MLflow code is:
+
+ ```bash
+ def log_into_mlflow(self):
+ # Set the MLflow tracking URI from the configuration
+ mlflow.set_registry_uri(self.config.mlflow_uri)
+
+ # Parse the tracking URI to determine the type of storage used
+ tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme
+
+ # Set the experiment name
+ experiment_name = "Model Evaluation Experimentation"
+ mlflow.set_experiment(experiment_name)
+
+ # Start a new MLflow run to log parameters and metrics
+ with mlflow.start_run(run_name="Model Evaluation"):
+ # Log parameters from the configuration
+ mlflow.log_params(self.config.all_params)
+ # Log evaluation metrics (loss and accuracy)
+ mlflow.log_metrics(
+ {"loss": self.score[0], "accuracy": self.score[1]}
+ )
+ # Check if the tracking URL is not a file store
+ if tracking_url_type_store != "file":
+ # Register the model with MLflow
+ mlflow.keras.log_model(self.model, "model", registered_model_name="VGG16Model")
+
+ else:
+ # Log the model without registration if using file store
+ mlflow.keras.log_model(self.model, "model")
+ ```
+
+ - Now to integrate **DVC** for **Pipeline Tracking** () use these commands:
+
+ - First step is to initialize DVC:
+
+ ```bash
+ dvc init
+ ```
+
+ - Second step will be to run this command for pipeline tracking which will run dvc.yaml to perform this:
+
+ ```bash
+ dvc repro
+ ```
+
+ - Now to see graph of how the whole pipeline connected (dependecy of the pipeline) run this command:
+
+ ```bash
+ dvc dag
+ ```
+
diff --git a/research/01_data_ingestion.ipynb b/research/01_data_ingestion.ipynb
index 949ace5..b0e408f 100644
--- a/research/01_data_ingestion.ipynb
+++ b/research/01_data_ingestion.ipynb
@@ -15,6 +15,13 @@
"\n"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### **Check Current Directory**"
+ ]
+ },
{
"cell_type": "code",
"execution_count": 1,
@@ -36,6 +43,13 @@
"%pwd"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### **Get 1 Folder Back**"
+ ]
+ },
{
"cell_type": "code",
"execution_count": 2,
@@ -45,9 +59,16 @@
"os.chdir(\"../\")"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### **Now getting Correct path for Operation**"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
"outputs": [
{
@@ -62,13 +83,12 @@
}
],
"source": [
- "\n",
"%pwd"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -76,13 +96,13 @@
"from dataclasses import dataclass\n",
"from pathlib import Path\n",
"\n",
- "\n",
+ "# creating an entity, frozen=True mean that no more elements can be added \n",
"@dataclass(frozen=True)\n",
"class DataIngestionConfig:\n",
" root_dir: Path\n",
" source_URL: str\n",
" local_data_file: Path\n",
- " unzip_dir: Path"
+ " unzip_dir: Path"
]
},
{
@@ -97,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -109,13 +129,15 @@
" config_filepath = CONFIG_FILE_PATH,\n",
" params_filepath = PARAMS_FILE_PATH):\n",
"\n",
+ " # read yaml files\n",
" self.config = read_yaml(config_filepath)\n",
" self.params = read_yaml(params_filepath)\n",
- "\n",
+ " \n",
+ " # create a directory to store data \n",
" create_directories([self.config.artifacts_root])\n",
"\n",
"\n",
- " \n",
+ " # data ingestion configuration\n",
" def get_data_ingestion_config(self) -> DataIngestionConfig:\n",
" config = self.config.data_ingestion\n",
"\n",
diff --git a/research/02_prepare_base_model.ipynb b/research/02_prepare_base_model.ipynb
index be067e3..45136b3 100644
--- a/research/02_prepare_base_model.ipynb
+++ b/research/02_prepare_base_model.ipynb
@@ -28,8 +28,7 @@
],
"source": [
"# check current directory\n",
- "!pwd\n",
- "!"
+ "!pwd"
]
},
{
@@ -435,34 +434,6 @@
" raise e"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/research/03_model_trainer.ipynb b/research/03_model_trainer.ipynb
index 1f2ce21..e0e0c1d 100644
--- a/research/03_model_trainer.ipynb
+++ b/research/03_model_trainer.ipynb
@@ -327,20 +327,6 @@
" raise e"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/research/experiments.ipynb b/research/experiments.ipynb
index 13069cd..723e6c0 100644
--- a/research/experiments.ipynb
+++ b/research/experiments.ipynb
@@ -16,54 +16,199 @@
"----\n"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### **config-box**\n",
+ "\n",
+ "- mostly used for yaml files for easy access"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 27,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "d = {\"key\":\"vaulue\",\"tala\":\"chabi\"}"
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 28,
"metadata": {},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'vaulue'"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "d[\"key\"]"
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 29,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "# d.key # will raise error "
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 30,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "# to access value like this use config-box\n",
+ "\n",
+ "from box import ConfigBox\n",
+ "\n",
+ "# let's define a dictionary\n",
+ "dic = {\"key\":\"vaulue\",\"tala\":\"chabi\"}\n",
+ "\n",
+ "# convert our dictionaary into configbox\n",
+ "dic = ConfigBox(dic)"
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 31,
"metadata": {},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'chabi'"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# now we can access values\n",
+ "dic.tala"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### **ensure**"
+ ]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "54"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def get_multiplication(x:int,y:int):\n",
+ " return x * y\n",
+ "\n",
+ "get_multiplication(x = 6, y=9)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'999999'"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# suppose i change data type of y\n",
+ "get_multiplication(x = 6, y=\"9\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "# to deal with exact data types we use this\n",
+ "from ensure import ensure_annotations\n",
+ "\n",
+ "@ensure_annotations\n",
+ "def get_multiplication(x:int,y:int):\n",
+ " return x * y"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "54"
+ ]
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# let me call function again\n",
+ "get_multiplication(x = 6, y=9)"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "ename": "EnsureError",
+ "evalue": "Argument y of type to does not match annotation type ",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mEnsureError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[1;32mIn[36], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# let me call function again\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mget_multiplication\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m6\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m9\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[1;32me:\\Practice python\\Chest Cancer Classification Using MLflow and DVC\\venv\\lib\\site-packages\\ensure\\main.py:833\u001b[0m, in \u001b[0;36mWrappedFunction.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 831\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, templ):\n\u001b[0;32m 832\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mArgument \u001b[39m\u001b[38;5;132;01m{arg}\u001b[39;00m\u001b[38;5;124m of type \u001b[39m\u001b[38;5;132;01m{valt}\u001b[39;00m\u001b[38;5;124m to \u001b[39m\u001b[38;5;132;01m{f}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdoes not match annotation type \u001b[39m\u001b[38;5;132;01m{t}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 833\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m EnsureError(msg\u001b[38;5;241m.\u001b[39mformat(arg\u001b[38;5;241m=\u001b[39marg, f\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mf, t\u001b[38;5;241m=\u001b[39mtempl, valt\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mtype\u001b[39m(value)))\n\u001b[0;32m 835\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mf(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+ "\u001b[1;31mEnsureError\u001b[0m: Argument y of type to does not match annotation type "
+ ]
+ }
+ ],
+ "source": [
+ "# let me call function again\n",
+ "get_multiplication(x = 6, y=\"9\") # it will throw error because data type do not match"
+ ]
},
{
"cell_type": "markdown",
diff --git a/scores.json b/scores.json
index d701995..1e0de55 100644
--- a/scores.json
+++ b/scores.json
@@ -1,4 +1,4 @@
{
- "loss": 0.0764155164361,
- "accuracy": 0.970588207244873
+ "loss": 0.7865707874298096,
+ "accuracy": 0.5972222089767456
}
\ No newline at end of file
diff --git a/src/chest_cancer_classifier/components/model_evaluation_with_mlflow.py b/src/chest_cancer_classifier/components/model_evaluation_with_mlflow.py
index d73fd8a..5726fb9 100644
--- a/src/chest_cancer_classifier/components/model_evaluation_with_mlflow.py
+++ b/src/chest_cancer_classifier/components/model_evaluation_with_mlflow.py
@@ -1,4 +1,5 @@
+# Import necessary libraries
import mlflow
import mlflow.keras
import tensorflow as tf
@@ -8,90 +9,219 @@
from chest_cancer_classifier.utils.common_functions import read_yaml, create_directories, save_json
from chest_cancer_classifier.entity.config_entity import EvaluationConfig
-
class Evaluation:
def __init__(self, config: EvaluationConfig):
- # Initialize the Evaluation class with the provided configuration
+ """
+ Initialize the Evaluation class with the provided configuration.
+
+ :param config: Configuration object containing evaluation settings.
+ """
self.config = config
def _valid_generator(self):
- # Create a data generator for validation images with specific configurations
+ """
+ Create a validation data generator to preprocess images for evaluation.
+ """
+ # Arguments for data normalization
datagenerator_kwargs = dict(
- rescale=1./255, # Normalize pixel values to [0, 1]
- validation_split=0.30 # Use 30% of the data for validation
+ rescale=1.0 / 255 # Normalize pixel values to [0, 1]
)
- # Specify parameters for the data flow generator
+ # Arguments for image resizing and batching
dataflow_kwargs = dict(
- target_size=self.config.params_image_size[:-1], # Resize images to the target size, excluding the last dimension (channels)
- batch_size=self.config.params_batch_size, # Set the batch size for loading images
- interpolation="bilinear" # Use bilinear interpolation for resizing
+ target_size=self.config.params_image_size[:-1], # Target size excluding channels
+ batch_size=self.config.params_batch_size, # Batch size
+ interpolation="bilinear" # Bilinear interpolation for resizing
)
# Create a Keras ImageDataGenerator for validation data
- valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
- **datagenerator_kwargs
- )
+ valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(**datagenerator_kwargs)
- # Generate batches of validation data from the specified directory
+ # Generate validation data batches from the specified directory
self.valid_generator = valid_datagenerator.flow_from_directory(
- directory=self.config.training_data, # Directory containing validation images
- subset="validation", # Specify that this is the validation subset
- shuffle=False, # Do not shuffle the data
- class_mode="sparse", # Set to "sparse" for integer labels
- **dataflow_kwargs # Pass the data flow parameters
+ directory=self.config.validation_data_dir, # Path to validation data directory
+ shuffle=False, # No shuffling for consistent evaluation
+ class_mode="sparse", # Use sparse mode for integer labels
+ **dataflow_kwargs
)
@staticmethod
def load_model(path: Path) -> tf.keras.Model:
- # Load a Keras model from the specified path
+ """
+ Load a Keras model from the specified file path.
+
+ :param path: Path to the model file.
+ :return: Loaded Keras model.
+ """
return tf.keras.models.load_model(path)
def evaluation(self):
- # Load the model specified in the configuration
+ """
+ Evaluate the model on the validation data and compute metrics.
+ """
+ # Load the trained model
self.model = self.load_model(self.config.path_of_model)
- # Recompile the model with binary_crossentropy to ensure correct configuration for binary classification
- self.model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
-
- # Create the validation data generator
+ # Compile the model with updated metrics
+ self.model.compile(
+ optimizer="adam", # Adam optimizer
+ loss="sparse_categorical_crossentropy", # Loss for multi-class classification
+ metrics=["accuracy"] # Updated metrics
+ )
+
+ # Prepare the validation data generator
self._valid_generator()
-
+
# Evaluate the model on the validation data
self.score = self.model.evaluate(self.valid_generator)
-
- # Save the evaluation scores
+
+ # Save evaluation scores
self.save_score()
def save_score(self):
- # Save the evaluation scores (loss and accuracy) to a JSON file
- scores = {"loss": self.score[0], "accuracy": self.score[1]}
+ """
+ Save the evaluation metrics to a JSON file.
+
+ :return: None
+ """
+ # Map evaluation metrics to their respective names
+ scores = {
+ "loss": self.score[0],
+ "accuracy": self.score[1],
+ }
+ # Save scores to a JSON file
save_json(path=Path("scores.json"), data=scores)
def log_into_mlflow(self):
- # Set the MLflow tracking URI from the configuration
+ """
+ Log evaluation metrics and model to MLflow for tracking and versioning.
+ """
+ # Set the MLflow tracking URI
mlflow.set_registry_uri(self.config.mlflow_uri)
-
- # Parse the tracking URI to determine the type of storage used
+
+ # Determine the type of storage used for MLflow tracking
tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme
-
+
# Set the experiment name
- experiment_name = "Model Evaluation Experimentation"
+ experiment_name = "Multi Class Classification Model Evaluation"
mlflow.set_experiment(experiment_name)
- # Start a new MLflow run to log parameters and metrics
- with mlflow.start_run(run_name="Chest Cancer Classification"):
- # Log parameters from the configuration
+ # Start a new MLflow run
+ with mlflow.start_run(run_name="Multi Class Classification"):
+ # Log all hyperparameters from the configuration
mlflow.log_params(self.config.all_params)
- # Log evaluation metrics (loss and accuracy)
- mlflow.log_metrics(
- {"loss": self.score[0], "accuracy": self.score[1]}
- )
- # Check if the tracking URL is not a file store
+
+ # Log evaluation metrics
+ mlflow.log_metrics({
+ "loss": self.score[0],
+ "accuracy": self.score[1],
+ })
+
+ # Log the model depending on the tracking URL type
if tracking_url_type_store != "file":
- # Register the model with MLflow
+ # Register the model with MLflow if not using file-based tracking
mlflow.keras.log_model(self.model, "model", registered_model_name="VGG16Model")
-
else:
- # Log the model without registration if using file store
+ # Log the model without registration for file-based tracking
mlflow.keras.log_model(self.model, "model")
+
+ # End the MLflow run
+ mlflow.end_run()
+
+# # Import necessary libraries
+# import mlflow
+# import mlflow.keras
+# import tensorflow as tf
+# from pathlib import Path
+# from urllib.parse import urlparse
+# from chest_cancer_classifier.constants import *
+# from chest_cancer_classifier.utils.common_functions import read_yaml, create_directories, save_json
+# from chest_cancer_classifier.entity.config_entity import EvaluationConfig
+
+
+# class Evaluation:
+# def __init__(self, config: EvaluationConfig):
+# # Initialize the Evaluation class with the provided configuration
+# self.config = config
+
+# def _valid_generator(self):
+# # Create a data generator for validation images with specific configurations
+# datagenerator_kwargs = dict(
+# rescale=1./255, # Normalize pixel values to [0, 1]
+# validation_split=0.30 # Use 30% of the data for validation
+# )
+
+# # Specify parameters for the data flow generator
+# dataflow_kwargs = dict(
+# target_size=self.config.params_image_size[:-1], # Resize images to the target size, excluding the last dimension (channels)
+# batch_size=self.config.params_batch_size, # Set the batch size for loading images
+# interpolation="bilinear" # Use bilinear interpolation for resizing
+# )
+
+# # Create a Keras ImageDataGenerator for validation data
+# valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
+# **datagenerator_kwargs
+# )
+
+# # Generate batches of validation data from the specified directory
+# self.valid_generator = valid_datagenerator.flow_from_directory(
+# directory=self.config.training_data, # Directory containing validation images
+# subset="validation", # Specify that this is the validation subset
+# shuffle=False, # Do not shuffle the data
+# class_mode="sparse", # Set to "sparse" for integer labels
+# **dataflow_kwargs # Pass the data flow parameters
+# )
+
+# @staticmethod
+# def load_model(path: Path) -> tf.keras.Model:
+# # Load a Keras model from the specified path
+# return tf.keras.models.load_model(path)
+
+# def evaluation(self):
+# # Load the model specified in the configuration
+# self.model = self.load_model(self.config.path_of_model)
+
+# # Recompile the model with binary_crossentropy to ensure correct configuration for binary classification
+# self.model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
+
+# # Create the validation data generator
+# self._valid_generator()
+
+# # Evaluate the model on the validation data
+# self.score = self.model.evaluate(self.valid_generator)
+
+# # Save the evaluation scores
+# self.save_score()
+
+# def save_score(self):
+# # Save the evaluation scores (loss and accuracy) to a JSON file
+# scores = {"loss": self.score[0], "accuracy": self.score[1]}
+# save_json(path=Path("scores.json"), data=scores)
+
+# def log_into_mlflow(self):
+# # Set the MLflow tracking URI from the configuration
+# mlflow.set_registry_uri(self.config.mlflow_uri)
+
+# # Parse the tracking URI to determine the type of storage used
+# tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme
+
+# # Set the experiment name
+# experiment_name = "Multi Class Classification Model Evaluation"
+# mlflow.set_experiment(experiment_name)
+
+# # Start a new MLflow run to log parameters and metrics
+# with mlflow.start_run(run_name="Multi Class Classification"):
+# # Log parameters from the configuration
+# mlflow.log_params(self.config.all_params)
+# # Log evaluation metrics (loss and accuracy)
+# mlflow.log_metrics(
+# {"loss": self.score[0], "accuracy": self.score[1]}
+# )
+# # Check if the tracking URL is not a file store
+# if tracking_url_type_store != "file":
+# # Register the model with MLflow
+# mlflow.keras.log_model(self.model, "model", registered_model_name="VGG16Model")
+
+# else:
+# # Log the model without registration if using file store
+# mlflow.keras.log_model(self.model, "model")
diff --git a/src/chest_cancer_classifier/components/model_trainer.py b/src/chest_cancer_classifier/components/model_trainer.py
index 619ddd1..4f09433 100644
--- a/src/chest_cancer_classifier/components/model_trainer.py
+++ b/src/chest_cancer_classifier/components/model_trainer.py
@@ -1,5 +1,6 @@
# import libraries
+
import os
import time
import tensorflow as tf
@@ -8,95 +9,223 @@
import urllib.request as request
from chest_cancer_classifier.entity.config_entity import TrainingConfig
+# for multi class classification
+
class Training:
def __init__(self, config: TrainingConfig):
- # Initialize the Training class with a TrainingConfig object
+ """
+ Initialize the Training class with a TrainingConfig object.
+
+ :param config: TrainingConfig object containing configuration for training.
+ """
self.config = config
def get_base_model(self):
- # Load the base model from the specified path in the configuration
+ """
+ Load and compile the base model for training.
+ """
+ # Load the pre-trained base model
self.model = tf.keras.models.load_model(self.config.updated_base_model_path)
- # Compile the model with an optimizer and loss function suitable for the task
+ # Compile the model with optimizer, loss, and metrics
self.model.compile(
- optimizer=tf.keras.optimizers.Adam(), # Using Adam optimizer for training
- loss='sparse_categorical_crossentropy', # Loss function for multi-class classification with integer labels
- metrics=['accuracy'] # Track accuracy as a performance metric
+ optimizer=tf.keras.optimizers.Adam(), # Adam optimizer for training
+ loss='sparse_categorical_crossentropy', # Loss function for classification
+ metrics = ["accuracy"] # Metrics to track model performance
)
def train_valid_generator(self):
- # Set up data generator arguments for preprocessing the images
+ """
+ Prepare training, validation, and test data generators.
+ """
+ # Arguments for data preprocessing
datagenerator_kwargs = dict(
- rescale=1.0 / 255, # Normalize the pixel values to the range [0, 1]
- validation_split=0.20 # Use 20% of the data for validation
+ rescale=1.0 / 255 # Normalize pixel values to [0, 1]
)
- # Set up data flow arguments for resizing images and defining batch size
+ # Arguments for image resizing and batching
dataflow_kwargs = dict(
- target_size=self.config.params_image_size[:-1], # Resize images to specified dimensions (excluding channels)
- batch_size=self.config.params_batch_size, # Set the batch size for training and validation
- interpolation="bilinear" # Set the interpolation method for resizing images
+ target_size=self.config.params_image_size[:-1], # Image size excluding channels
+ batch_size=self.config.params_batch_size, # Batch size for generators
+ interpolation="bilinear" # Interpolation method for resizing images
)
- # Create a validation data generator
+ # Validation data generator using 'valid' folder
valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
**datagenerator_kwargs
)
self.valid_generator = valid_datagenerator.flow_from_directory(
- directory=self.config.training_data, # Directory containing training and validation data
- subset="validation", # Specify that this generator is for validation data
- shuffle=False, # Do not shuffle validation data
- class_mode="sparse", # Use "sparse" mode for integer labels
- **dataflow_kwargs # Include data flow arguments for resizing and batch size
+ directory=os.path.join(self.config.training_data, "valid"), # Path to validation data
+ shuffle=False, # No shuffling for validation data
+ class_mode="sparse", # Sparse mode for integer labels
+ **dataflow_kwargs
)
- # Create a training data generator with or without augmentation
+ # Training data generator
if self.config.params_is_augmentation:
- # If augmentation is enabled, configure the data generator with augmentation techniques
+ # Augmentation for training data if enabled in config
train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
- rotation_range=40, # Randomly rotate images within a range
- horizontal_flip=True, # Randomly flip images horizontally
- width_shift_range=0.2, # Randomly shift images horizontally
- height_shift_range=0.2, # Randomly shift images vertically
- shear_range=0.2, # Apply random shearing transformations
- zoom_range=0.2, # Randomly zoom into images
- **datagenerator_kwargs # Include normalization and validation split
+ rotation_range=40, # Random rotation
+ horizontal_flip=True, # Random horizontal flips
+ width_shift_range=0.2, # Horizontal shift
+ height_shift_range=0.2, # Vertical shift
+ shear_range=0.2, # Shear transformation
+ zoom_range=0.2, # Zoom transformation
+ **datagenerator_kwargs
)
else:
- # If no augmentation is needed, use the validation data generator
- train_datagenerator = valid_datagenerator
+ # Simple generator without augmentation
+ train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
+ **datagenerator_kwargs
+ )
- # Generate training data from the directory
self.train_generator = train_datagenerator.flow_from_directory(
- directory=self.config.training_data, # Directory containing training data
- subset="training", # Specify that this generator is for training data
+ directory=os.path.join(self.config.training_data, "train"), # Path to training data
shuffle=True, # Shuffle training data
- class_mode="sparse", # Use "sparse" mode for integer labels
- **dataflow_kwargs # Include data flow arguments for resizing and batch size
+ class_mode="sparse", # Sparse mode for integer labels
+ **dataflow_kwargs
+ )
+
+ # Test data generator using 'test' folder
+ test_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
+ **datagenerator_kwargs
+ )
+ self.test_generator = test_datagenerator.flow_from_directory(
+ directory=os.path.join(self.config.training_data, "test"), # Path to test data
+ shuffle=False, # No shuffling for test data
+ class_mode="sparse", # Sparse mode for integer labels
+ **dataflow_kwargs
)
@staticmethod
def save_model(path: Path, model: tf.keras.Model):
- # Static method to save the trained model to the specified path
+ """
+ Save the trained model to the specified path.
+
+ :param path: Path to save the model.
+ :param model: Trained model object.
+ """
model.save(path)
def train(self):
- # Calculate the number of steps per epoch based on training data
+ """
+ Train the model using the training and validation data generators.
+ """
+ # Calculate steps per epoch for training and validation
self.steps_per_epoch = self.train_generator.samples // self.train_generator.batch_size
- # Calculate the number of validation steps based on validation data
self.validation_steps = self.valid_generator.samples // self.valid_generator.batch_size
- # Start the training process
+ # Train the model
self.model.fit(
self.train_generator, # Training data generator
- epochs=self.config.params_epochs, # Number of epochs specified in the configuration
+ epochs=self.config.params_epochs, # Number of epochs
steps_per_epoch=self.steps_per_epoch, # Steps per epoch
- validation_steps=self.validation_steps, # Steps for validation
- validation_data=self.valid_generator # Validation data generator
+ validation_data=self.valid_generator, # Validation data generator
+ validation_steps=self.validation_steps # Validation steps
)
- # Save the trained model to the specified path
+ # Save the trained model
self.save_model(
- path=self.config.trained_model_path, # Path where the trained model will be saved
- model=self.model # The trained model to save
- )
\ No newline at end of file
+ path=self.config.trained_model_path, # Save path for trained model
+ model=self.model # The trained model
+ )
+
+
+
+
+
+
+## For binary classification
+
+# class Training:
+# def __init__(self, config: TrainingConfig):
+# # Initialize the Training class with a TrainingConfig object
+# self.config = config
+
+# def get_base_model(self):
+# # Load the base model from the specified path in the configuration
+# self.model = tf.keras.models.load_model(self.config.updated_base_model_path)
+
+# # Compile the model with an optimizer and loss function suitable for the task
+# self.model.compile(
+# optimizer=tf.keras.optimizers.Adam(), # Using Adam optimizer for training
+# loss='sparse_categorical_crossentropy', # Loss function for multi-class classification with integer labels
+# metrics = ["accuracy", "precision", "recall", "f1_score"] # Track accuracy as a performance metric
+# )
+
+# def train_valid_generator(self):
+# # Set up data generator arguments for preprocessing the images
+# datagenerator_kwargs = dict(
+# rescale=1.0 / 255, # Normalize the pixel values to the range [0, 1]
+# validation_split=0.20 # Use 20% of the data for validation
+# )
+
+# # Set up data flow arguments for resizing images and defining batch size
+# dataflow_kwargs = dict(
+# target_size=self.config.params_image_size[:-1], # Resize images to specified dimensions (excluding channels)
+# batch_size=self.config.params_batch_size, # Set the batch size for training and validation
+# interpolation="bilinear" # Set the interpolation method for resizing images
+# )
+
+# # Create a validation data generator
+# valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
+# **datagenerator_kwargs
+# )
+# self.valid_generator = valid_datagenerator.flow_from_directory(
+# directory=self.config.training_data, # Directory containing training and validation data
+# subset="validation", # Specify that this generator is for validation data
+# shuffle=False, # Do not shuffle validation data
+# class_mode="sparse", # Use "sparse" mode for integer labels
+# **dataflow_kwargs # Include data flow arguments for resizing and batch size
+# )
+
+# # Create a training data generator with or without augmentation
+# if self.config.params_is_augmentation:
+# # If augmentation is enabled, configure the data generator with augmentation techniques
+# train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(
+# rotation_range=40, # Randomly rotate images within a range
+# horizontal_flip=True, # Randomly flip images horizontally
+# width_shift_range=0.2, # Randomly shift images horizontally
+# height_shift_range=0.2, # Randomly shift images vertically
+# shear_range=0.2, # Apply random shearing transformations
+# zoom_range=0.2, # Randomly zoom into images
+# **datagenerator_kwargs # Include normalization and validation split
+# )
+# else:
+# # If no augmentation is needed, use the validation data generator
+# train_datagenerator = valid_datagenerator
+
+# # Generate training data from the directory
+# self.train_generator = train_datagenerator.flow_from_directory(
+# directory=self.config.training_data, # Directory containing training data
+# subset="training", # Specify that this generator is for training data
+# shuffle=True, # Shuffle training data
+# class_mode="sparse", # Use "sparse" mode for integer labels
+# **dataflow_kwargs # Include data flow arguments for resizing and batch size
+# )
+
+# @staticmethod
+# def save_model(path: Path, model: tf.keras.Model):
+# # Static method to save the trained model to the specified path
+# model.save(path)
+
+# def train(self):
+# # Calculate the number of steps per epoch based on training data
+# self.steps_per_epoch = self.train_generator.samples // self.train_generator.batch_size
+# # Calculate the number of validation steps based on validation data
+# self.validation_steps = self.valid_generator.samples // self.valid_generator.batch_size
+
+# # Start the training process
+# self.model.fit(
+# self.train_generator, # Training data generator
+# epochs=self.config.params_epochs, # Number of epochs specified in the configuration
+# steps_per_epoch=self.steps_per_epoch, # Steps per epoch
+# validation_steps=self.validation_steps, # Steps for validation
+# validation_data=self.valid_generator # Validation data generator
+# )
+
+# # Save the trained model to the specified path
+# self.save_model(
+# path=self.config.trained_model_path, # Path where the trained model will be saved
+# model=self.model # The trained model to save
+# )
\ No newline at end of file
diff --git a/src/chest_cancer_classifier/components/prepare_base_models.py b/src/chest_cancer_classifier/components/prepare_base_models.py
index 6d37115..988b140 100644
--- a/src/chest_cancer_classifier/components/prepare_base_models.py
+++ b/src/chest_cancer_classifier/components/prepare_base_models.py
@@ -58,7 +58,7 @@ def _prepare_full_model(model, classes, freeze_all, freeze_till, learning_rate):
full_model.compile(
optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate), # Stochastic Gradient Descent optimizer
loss=tf.keras.losses.CategoricalCrossentropy(), # Loss function for multi-class classification
- metrics=["accuracy"] # Track accuracy during training
+ metrics = ["accuracy"] # Track accuracy during training with these 3 metrics
)
# Print the model summary to the console
diff --git a/src/chest_cancer_classifier/config/configuration.py b/src/chest_cancer_classifier/config/configuration.py
index b311c2a..2d04d88 100644
--- a/src/chest_cancer_classifier/config/configuration.py
+++ b/src/chest_cancer_classifier/config/configuration.py
@@ -75,7 +75,7 @@ def get_training_config(self) -> TrainingConfig:
params = self.params
# Define the path to the training data
- training_data = os.path.join(self.config.data_ingestion.unzip_dir, "Chest-CT-Scan-data")
+ training_data = os.path.join(self.config.data_ingestion.unzip_dir, "Data")
# Create necessary directories for training
create_directories([
@@ -102,7 +102,9 @@ def get_evaluation_config(self) -> EvaluationConfig:
# Initialize the EvaluationConfig with relevant parameters
eval_config = EvaluationConfig(
path_of_model="artifacts/training/model.h5",
- training_data="artifacts/data_ingestion/Chest-CT-Scan-data",
+ # training_data="artifacts/data_ingestion/Chest-CT-Scan-data",
+ validation_data_dir="artifacts/data_ingestion/Data/valid",
+ training_data="artifacts/data_ingestion/Data/train",
mlflow_uri="https://dagshub.com/muhammadadilnaeem/Chest-Cancer-Classification-Using-MLflow-and-DVC.mlflow",
all_params=self.params,
params_image_size=self.params.IMAGE_SIZE,
diff --git a/src/chest_cancer_classifier/entity/config_entity.py b/src/chest_cancer_classifier/entity/config_entity.py
index e76848f..323162b 100644
--- a/src/chest_cancer_classifier/entity/config_entity.py
+++ b/src/chest_cancer_classifier/entity/config_entity.py
@@ -40,6 +40,7 @@ class TrainingConfig:
class EvaluationConfig:
path_of_model: Path # Path to the model to be evaluated
training_data: Path # Path to the training dataset used for evaluation
+ validation_data_dir: Path # Path to the validation dataset directory
all_params: dict # Dictionary containing all relevant parameters for evaluation
mlflow_uri: str # URI for MLflow tracking server
params_image_size: list # Image dimensions for input to the model
diff --git a/static/style.css b/static/style.css
deleted file mode 100644
index e69de29..0000000
diff --git a/templates/index.html b/templates/index.html
deleted file mode 100644
index 7d7bda8..0000000
--- a/templates/index.html
+++ /dev/null
@@ -1,337 +0,0 @@
-
-
-
-
-
-
-
- Chest Cancer Classifier
-
-
-
-
-
-
-
- AI-Powered Chest Cancer Detection
- 
-
-
-