init commit

.gitignore

@@ -0,0 +1,63 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+.env
+.venv
+env/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+.project
+.pydevproject
+.settings/
+
+# Project specific
+temp_data/
+*.bak
+*.bacpac
+*.dump
+*.sql
+vpic.db
+logs/
+
+# Docker
+.docker/
+docker-compose.override.yml
+
+# OS specific
+.DS_Store
+Thumbs.db
+*.log
+
+# Test coverage
+.coverage
+htmlcov/
+.pytest_cache/
+.tox/
+coverage.xml
+*.cover 

Makefile

@@ -0,0 +1,92 @@
+# Makefile
+.PHONY: all clean setup install start-containers download restore migrate-pg migrate-sqlite verify backup export-pg test
+
+# Variables
+BACKUP_NAME := vpic_postgres_$(shell date +%Y%m%d_%H%M%S)
+TEMP_DIR := temp_data
+SQL_CONTAINER := sqltemp
+PG_CONTAINER := pg_target
+VENV := .venv
+PYTHON := $(VENV)/bin/python
+PIP := $(VENV)/bin/pip
+
+# Default target
+all: clean setup start-containers download restore migrate-pg verify backup
+
+# Clean environment
+clean:
+	@echo "Cleaning environment..."
+	docker-compose -f docker/docker-compose.yml down -v || true
+	rm -rf $(VENV) *.egg-info dist build __pycache__ vpic_migration/__pycache__
+
+# Setup Python virtual environment
+$(VENV)/bin/activate: requirements.txt
+	python -m venv $(VENV)
+	$(PIP) install -U pip setuptools wheel
+	$(PIP) install -r requirements.txt
+	$(PYTHON) setup.py develop
+
+# Setup environment
+setup: $(VENV)/bin/activate
+
+# Install package in development mode
+install: setup
+	$(PYTHON) setup.py develop
+
+# Start Docker containers
+start-containers:
+	@echo "Starting containers..."
+	docker-compose -f docker/docker-compose.yml up -d
+	@echo "Waiting for containers to be ready..."
+	sleep 20
+
+# Download vPIC data
+download:
+	@echo "Downloading vPIC data..."
+	./scripts/download_vpic.sh
+
+# Restore SQL Server backup
+restore:
+	@echo "Restoring SQL Server backup..."
+	./scripts/restore_backup.sh
+	./scripts/verify_db.sh
+
+# Migrate to PostgreSQL
+migrate-pg: install
+	@echo "Migrating to PostgreSQL..."
+	TARGET_DB=postgres $(PYTHON) -m vpic_migration.migrate
+
+# Migrate to SQLite
+migrate-sqlite: install
+	@echo "Migrating to SQLite..."
+	TARGET_DB=sqlite $(PYTHON) -m vpic_migration.migrate
+
+# Verify migration
+verify-pg:
+	@echo "Verifying migration..."
+	@echo "SQL Server tables:"
+	./scripts/verify_db.sh
+	@echo "PostgreSQL tables:"
+	docker exec $(PG_CONTAINER) psql -U postgres -d vpic -c "\dt+"
+
+# Run tests
+test: install
+	@echo "Running tests..."
+	$(PYTHON) -m pytest tests/
+
+# Create backup
+backup: export-pg
+
+# Export PostgreSQL database
+export-pg:
+	@echo "Exporting PostgreSQL database..."
+	mkdir -p $(TEMP_DIR)
+	@echo "Creating schema-only backup..."
+	docker exec $(PG_CONTAINER) pg_dump -U postgres -d vpic --schema-only > $(TEMP_DIR)/$(BACKUP_NAME)_schema.sql
+	@echo "Creating data-only backup..."
+	docker exec $(PG_CONTAINER) pg_dump -U postgres -d vpic --data-only > $(TEMP_DIR)/$(BACKUP_NAME)_data.sql
+	@echo "Creating complete backup..."
+	docker exec $(PG_CONTAINER) pg_dump -U postgres -d vpic -Fc > $(TEMP_DIR)/$(BACKUP_NAME).dump
+	@echo "Backup files created in $(TEMP_DIR):"
+	@ls -lh $(TEMP_DIR)/$(BACKUP_NAME)*
+

README.md

@@ -0,0 +1,122 @@
+# vPIC Database Migration Tool
+
+A robust tool for downloading, migrating, and managing the NHTSA's Vehicle Product Information Catalog (vPIC) database across different database platforms (SQL Server, PostgreSQL, and SQLite).
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Overview
+
+This tool facilitates the migration of the NHTSA's vPIC database, which contains comprehensive vehicle specification data, including:
+- Vehicle Makes, Models, and Types
+- Manufacturer Information
+- Vehicle Specifications and Features
+- WMI (World Manufacturer Identifier) Data
+- VIN Decoder implementation
+
+## Features
+
+- 🚀 Automated download of the latest vPIC database backup
+- 🔄 Migration support for multiple database platforms:
+  - Microsoft SQL Server
+  - PostgreSQL
+  - SQLite
+- ✅ Data integrity verification
+- 📊 Progress tracking with detailed logging
+- 🔧 Configurable settings and type mappings
+- 🐳 Docker support for easy deployment
+
+## Prerequisites
+
+- Python 3.8 or higher
+- Docker and Docker Compose
+- Make (optional, but recommended)
+
+### System Requirements
+
+- Storage: At least 10GB free space (varies based on data size)
+- Memory: Minimum 4GB RAM recommended
+- OS: Compatible with Linux, macOS, and Windows
+
+## Quick Start
+
+1. Clone the repository:
+   git clone https://github.com/yourusername/vpic-migration.git
+   cd vpic-migration
+
+2. Install dependencies:
+   # On macOS
+   ./install_deps.sh
+
+   # On Windows / Linux
+   python -m venv .venv
+   .venv\Scripts\activate
+   pip install -r requirements.txt
+
+3. Start the containers:
+   make start-containers
+
+4. Run the migration:
+   make download
+   make restore
+   make migrate-pg
+   make migrate-sqlite
+   make verify-pg
+   make backup
+
+   or
+
+   make all
+
+## Usage
+
+### Basic Usage
+
+The simplest way to use the tool is through the provided Makefile commands:
+
+# Run all steps
+make all
+
+# Download latest vPIC data
+make download
+
+# Restore SQL Server backup
+make restore
+
+# Migrate to PostgreSQL
+make migrate-pg
+
+# Migrate to SQLite
+make migrate-sqlite
+
+# Verify migration
+make verify
+
+# Create backup
+make backup
+
+## Configuration
+
+Configuration can be modified through environment variables or by editing vpic_migration/settings.py:
+
+SQL_SERVER = {
+    "driver": "ODBC Driver 18 for SQL Server",
+    "server": "localhost",
+    "database": "vpic",
+    "user": "SA",
+    "password": "YourPassword",
+    "trust_cert": "yes"
+}
+
+See [CONFIGURATION.md](docs/CONFIGURATION.md) for all available options.
+
+## Data Structure
+
+The vPIC database contains numerous tables with vehicle-related information. Key tables include:
+
+- Make: Vehicle manufacturers
+- Model: Vehicle models
+- VehicleType: Types of vehicles
+- WMI: World Manufacturer Identifier information
+- And many more...
+
+For complete schema information, see [DATA_STRUCTURE.md](docs/DATA_STRUCTURE.md).

docker/docker-compose.yml

@@ -0,0 +1,89 @@
+version: '3.8'
+services:
+  sqlserver:
+    build:
+      context: .
+      dockerfile: sqledge.Dockerfile
+    container_name: sqltemp
+    environment:
+      - ACCEPT_EULA=1
+      - MSSQL_SA_PASSWORD=DevPassword123#
+      - MSSQL_PID=Developer
+      # Disable encryption requirement for local dev
+      - MSSQL_ENCRYPT=DISABLED
+    ports:
+      - "1433:1433"
+    volumes:
+      - ../temp_data:/var/opt/mssql/backup
+    healthcheck:
+      test: /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "DevPassword123#" -C -Q "SELECT 1" || exit 1
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+
+  postgres:
+    image: postgres:15
+    container_name: pg_target
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=vpic
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # postgres:
+  #   image: postgres:15
+  #   container_name: pg_target
+  #   environment:
+  #     - POSTGRES_USER=postgres
+  #     - POSTGRES_PASSWORD=postgres
+  #     - POSTGRES_DB=vpic
+  #   ports:
+  #     - "5432:5432"
+  #   volumes:
+  #     - postgres_data:/var/lib/postgresql/data
+  #   command: 
+  #     - postgres
+  #     - "-c"
+  #     - "listen_addresses=*"
+  #     - "-c"
+  #     - "password_encryption=md5"
+
+  #   healthcheck:
+  #     test: ["CMD-SHELL", "pg_isready -U postgres"]
+  #     interval: 10s
+  #     timeout: 5s
+  #     retries: 5
+
+  # postgres:
+  #   image: postgres:15
+  #   container_name: pg_target
+  #   environment:
+  #     - POSTGRES_USER=postgres
+  #     - POSTGRES_PASSWORD=postgres
+  #     - POSTGRES_DB=vpic
+  #   ports:
+  #     - "5432:5432"
+  #   volumes:
+  #     - postgres_data:/var/lib/postgresql/data
+  #   command: 
+  #     - "postgres"
+  #     - "-c"
+  #     - "password_encryption=md5"
+  #   healthcheck:
+  #     test: ["CMD-SHELL", "pg_isready -U postgres"]
+  #     interval: 10s
+  #     timeout: 5s
+  #     retries: 5
+
+volumes:
+  postgres_data:

docker/sqledge.Dockerfile

@@ -0,0 +1,26 @@
+# ./docker/sqledge.Dockerfile
+FROM mcr.microsoft.com/azure-sql-edge:latest
+
+USER root
+
+# Create required directories and set permissions
+RUN mkdir -p /var/lib/apt/lists/partial && \
+  chmod 755 /var/lib/apt/lists/partial
+
+# Install prerequisites and SQL tools
+RUN apt-get update && \
+  apt-get install -y curl gnupg2 unixodbc-dev && \
+  curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
+  curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y mssql-tools18 msodbcsql18 && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Add SQL tools to PATH
+ENV PATH="/opt/mssql-tools18/bin:${PATH}"
+
+# Switch back to mssql user
+USER mssql
+
+CMD [ "/opt/mssql/bin/sqlservr" ]