AI-Powered Predictive Healthcare System for Disease Outbreak Detection Python
👤 Sharing: AI
```python
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier # Or other classifiers like LogisticRegression, GradientBoostingClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
from sklearn.preprocessing import LabelEncoder, StandardScaler
import matplotlib.pyplot as plt
import seaborn as sns
# --- 1. Data Loading and Preprocessing ---
def load_and_preprocess_data(filepath):
"""
Loads data from a CSV file, preprocesses it, and prepares it for machine learning.
Args:
filepath (str): The path to the CSV file.
Returns:
tuple: A tuple containing:
- X (pd.DataFrame): The feature matrix.
- y (pd.Series): The target variable (disease outbreak indicator).
- encoder (LabelEncoder): The LabelEncoder used for encoding target variable.
- scaler (StandardScaler): The StandardScaler used for feature scaling.
"""
try:
data = pd.read_csv(filepath)
except FileNotFoundError:
print(f"Error: File not found at {filepath}")
return None, None, None, None
# --- Data Cleaning (Handle Missing Values) ---
# Replace missing values with the mean for numerical columns.
for col in data.select_dtypes(include=np.number).columns:
data[col].fillna(data[col].mean(), inplace=True) #Or use median, or more sophisticated imputation
# Handle missing values in categorical columns with the mode.
for col in data.select_dtypes(include='object').columns:
data[col].fillna(data[col].mode()[0], inplace=True) # Mode handles potential multiple modes
# --- Feature Engineering (Example) ---
# Create a new feature (e.g., a composite health index based on existing features)
# This is an example, adapt to your specific data.
# Ensure to select columns that make sense for your domain. It's critical to understand your data.
if 'temperature' in data.columns and 'humidity' in data.columns:
data['temperature_humidity_index'] = data['temperature'] * data['humidity'] #Example
else:
print("Warning: 'temperature' or 'humidity' columns not found. Skipping temperature_humidity_index creation.")
# --- Encoding Categorical Features ---
# Use one-hot encoding for categorical features
categorical_cols = data.select_dtypes(include='object').columns
data = pd.get_dummies(data, columns=categorical_cols, dummy_na=False) # dummy_na=False to avoid creating a dummy variable for NaN
# --- Target Variable Encoding ---
# Assuming 'disease_outbreak' is the target variable (replace if needed)
target_column = 'disease_outbreak' # Replace with the actual column name
if target_column not in data.columns:
print(f"Error: Target column '{target_column}' not found.")
return None, None, None, None
# Encode the target variable (assuming it's categorical - e.g., 'Yes', 'No')
encoder = LabelEncoder()
data[target_column] = encoder.fit_transform(data[target_column]) # Converts labels to numbers (0, 1)
# --- Feature Scaling ---
# Separate features (X) and target (y)
y = data[target_column]
X = data.drop(target_column, axis=1) # Remove the target column from features
# Scale numerical features using StandardScaler
scaler = StandardScaler()
X = scaler.fit_transform(X) # Scale all numerical features
# Convert X back to DataFrame for easier handling (optional but recommended)
X = pd.DataFrame(X, columns=X.columns if isinstance(X, pd.DataFrame) else range(X.shape[1]))
X.columns = data.drop(target_column, axis=1).columns #Ensure the original column names are preserved
return X, y, encoder, scaler
# --- 2. Model Training ---
def train_model(X, y, model_type='random_forest', test_size=0.2, random_state=42):
"""
Trains a machine learning model.
Args:
X (pd.DataFrame): The feature matrix.
y (pd.Series): The target variable.
model_type (str): The type of model to train ('random_forest', 'logistic_regression', 'gradient_boosting').
test_size (float): The proportion of the data to use for testing.
random_state (int): Random seed for reproducibility.
Returns:
tuple: A tuple containing:
- model: The trained model.
- X_test (pd.DataFrame): The test feature matrix.
- y_test (pd.Series): The test target variable.
"""
# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state)
# --- Model Selection ---
if model_type == 'random_forest':
model = RandomForestClassifier(random_state=random_state) # Add hyperparameters for tuning (e.g., n_estimators, max_depth)
elif model_type == 'logistic_regression':
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(random_state=random_state, solver='liblinear') # Adjust solver as needed
elif model_type == 'gradient_boosting':
from sklearn.ensemble import GradientBoostingClassifier
model = GradientBoostingClassifier(random_state=random_state) # Add hyperparameters
else:
raise ValueError("Invalid model_type. Choose 'random_forest', 'logistic_regression', or 'gradient_boosting'.")
# --- Model Training ---
model.fit(X_train, y_train)
return model, X_test, y_test
# --- 3. Model Evaluation ---
def evaluate_model(model, X_test, y_test):
"""
Evaluates the trained model.
Args:
model: The trained model.
X_test (pd.DataFrame): The test feature matrix.
y_test (pd.Series): The test target variable.
Returns:
dict: A dictionary of evaluation metrics.
"""
y_pred = model.predict(X_test)
y_pred_proba = model.predict_proba(X_test)[:, 1] # Probabilities for ROC AUC
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
roc_auc = roc_auc_score(y_test, y_pred_proba)
metrics = {
'accuracy': accuracy,
'precision': precision,
'recall': recall,
'f1_score': f1,
'roc_auc': roc_auc
}
print("Evaluation Metrics:")
for metric, value in metrics.items():
print(f"{metric}: {value}")
return metrics
# --- 4. Prediction Function ---
def predict_outbreak(model, data, encoder, scaler):
"""
Predicts the probability of a disease outbreak for new data.
Args:
model: The trained model.
data (pd.DataFrame): New data for prediction. Must have same columns as the training data (after preprocessing).
encoder (LabelEncoder): The LabelEncoder used for target encoding during training.
scaler (StandardScaler): The StandardScaler used for feature scaling during training.
Returns:
float: The probability of a disease outbreak.
"""
# --- Data Preprocessing (same as training data!) ---
# Store the column names before any modifications
original_columns = data.columns.tolist()
# --- Data Cleaning (Handle Missing Values) ---
for col in data.select_dtypes(include=np.number).columns:
data[col].fillna(data[col].mean(), inplace=True)
for col in data.select_dtypes(include='object').columns:
data[col].fillna(data[col].mode()[0], inplace=True)
# --- Feature Engineering (same as training data!) ---
if 'temperature' in data.columns and 'humidity' in data.columns:
data['temperature_humidity_index'] = data['temperature'] * data['humidity']
# --- Encoding Categorical Features ---
categorical_cols = data.select_dtypes(include='object').columns
data = pd.get_dummies(data, columns=categorical_cols, dummy_na=False)
# --- Ensure correct columns ---
# This is CRUCIAL. Make sure the columns match the training data.
# You will likely need to load the training data columns from somewhere (e.g., saved file).
# This example assumes you know the columns that were used during training.
# Ensure that `training_columns` contains the exact columns used for training, including dummy variables
# Add a placeholder. **Replace this with the actual columns used during training.**
training_columns = ['temperature', 'humidity', 'population_density', 'previous_cases', 'health_index', 'location_A', 'location_B'] #Example!!! Replace!
# Add missing columns and fill with 0
for col in training_columns:
if col not in data.columns:
data[col] = 0 # Or another appropriate default value
# Select only the columns used during training and reorder
data = data[training_columns]
# --- Scaling ---
data_scaled = scaler.transform(data)
data_scaled = pd.DataFrame(data_scaled, columns=data.columns)
# --- Prediction ---
probability = model.predict_proba(data_scaled)[:, 1][0] # Get probability of outbreak (class 1)
return probability
# --- 5. Visualization (Example) ---
def visualize_feature_importance(model, feature_names):
"""
Visualizes feature importance for tree-based models.
Args:
model: The trained tree-based model (e.g., RandomForestClassifier).
feature_names (list): A list of feature names.
"""
if hasattr(model, 'feature_importances_'): # Check if the model has feature_importances_ attribute
importances = model.feature_importances_
feature_importances = pd.Series(importances, index=feature_names)
feature_importances = feature_importances.sort_values(ascending=False)
plt.figure(figsize=(10, 6))
sns.barplot(x=feature_importances, y=feature_importances.index)
plt.title("Feature Importance")
plt.xlabel("Importance Score")
plt.ylabel("Feature")
plt.show()
else:
print("Model does not support feature importance visualization.")
# --- 6. Main Execution ---
if __name__ == "__main__":
# --- 1. Load and Preprocess Data ---
data_file = 'disease_outbreak_data.csv' # Replace with your data file path
X, y, encoder, scaler = load_and_preprocess_data(data_file)
if X is None or y is None:
print("Data loading and preprocessing failed. Exiting.")
exit()
# --- 2. Train the Model ---
model, X_test, y_test = train_model(X, y, model_type='random_forest')
# --- 3. Evaluate the Model ---
evaluation_metrics = evaluate_model(model, X_test, y_test)
# --- 4. Visualize Feature Importance ---
visualize_feature_importance(model, X.columns)
# --- 5. Make a Prediction on New Data ---
# Create some dummy new data for prediction (replace with actual data)
new_data = pd.DataFrame({
'temperature': [30.0],
'humidity': [75.0],
'population_density': [1000],
'previous_cases': [50],
'health_index': [0.7],
'location': ['A'] #Example categorical feature. Make sure it is consistent with the training data.
})
outbreak_probability = predict_outbreak(model, new_data, encoder, scaler)
print(f"Predicted Probability of Outbreak: {outbreak_probability:.4f}")
# Explanation of the Code:
# 1. Data Loading and Preprocessing:
# - `load_and_preprocess_data(filepath)`: This function loads the data from a CSV file using pandas. It handles missing values by filling numerical columns with the mean and categorical columns with the mode.
# - Feature Engineering: An example is provided, but this is highly dependent on the specific dataset. Create new features that might be relevant for prediction.
# - One-Hot Encoding: Converts categorical features into numerical data using `pd.get_dummies`.
# - Label Encoding: Encodes the target variable into numerical values (0 and 1) using `LabelEncoder`.
# - Feature Scaling: Scales the numerical features using `StandardScaler` to improve model performance.
# 2. Model Training:
# - `train_model(X, y, model_type)`: Splits the data into training and testing sets using `train_test_split`.
# - Model Selection: Selects a model type (Random Forest, Logistic Regression, or Gradient Boosting) based on the `model_type` argument. You can easily add more models here.
# - Model Training: Trains the selected model using the training data.
# 3. Model Evaluation:
# - `evaluate_model(model, X_test, y_test)`: Evaluates the trained model using the test data.
# - Calculates accuracy, precision, recall, F1-score, and ROC AUC score.
# 4. Prediction:
# - `predict_outbreak(model, data, encoder, scaler)`: Predicts the probability of a disease outbreak for new data.
# - It's *crucial* that the new data is preprocessed in exactly the same way as the training data. This includes handling missing values, encoding categorical features, and scaling numerical features using the *same* `encoder` and `scaler` objects that were fitted on the training data.
# 5. Visualization:
# - `visualize_feature_importance(model, feature_names)`: Visualizes the feature importances for tree-based models. This helps understand which features are most important for the model's predictions.
# 6. Main Execution:
# - Loads the data, trains the model, evaluates the model, and makes a prediction on new data.
# - The `if __name__ == "__main__":` block ensures that this code only runs when the script is executed directly, not when it's imported as a module.
# Key Improvements and Considerations:
# * **Error Handling:** The code includes basic error handling (e.g., checking for file existence). Add more robust error handling, especially for data inconsistencies.
# * **Missing Value Imputation:** The code uses simple mean/mode imputation. Consider more sophisticated imputation techniques like k-Nearest Neighbors imputation or using a separate model to predict missing values.
# * **Feature Engineering:** This is where domain expertise is crucial. Think carefully about what features might be relevant for predicting disease outbreaks and create them. Examples include:
# * Time-series features (e.g., lagged values of disease cases).
# * Spatial features (e.g., proximity to hospitals, population centers).
# * Environmental factors (e.g., rainfall, temperature patterns).
# * Socioeconomic factors (e.g., poverty rates, access to healthcare).
# * **Data Validation:** Add data validation steps to ensure that the data is consistent and within expected ranges.
# * **Hyperparameter Tuning:** The code uses default hyperparameters for the models. Use techniques like grid search or randomized search to find the optimal hyperparameters for your data.
# * **Cross-Validation:** Use cross-validation to get a more robust estimate of the model's performance.
# * **Model Persistence:** Save the trained model, encoder, and scaler to files so that you can load them later without retraining. Use `pickle` or `joblib` for this.
# * **Explainable AI (XAI):** Use techniques like SHAP values or LIME to explain the model's predictions. This is important for building trust in the system and understanding why it's making certain predictions.
# * **Data Privacy:** If you are working with sensitive patient data, ensure that you are following all relevant data privacy regulations (e.g., HIPAA). Consider using techniques like differential privacy to protect patient privacy.
# * **Real-time Data Integration:** For a real-world system, you would need to integrate the model with a real-time data stream (e.g., from electronic health records, social media feeds, sensor data).
# * **Alerting System:** Develop an alerting system that triggers when the model predicts a high probability of a disease outbreak.
# * **Monitoring and Maintenance:** Continuously monitor the model's performance and retrain it as needed with new data.
# * **Version Control:** Use Git to track changes to your code.
# * **Documentation:** Write clear and comprehensive documentation for your code.
# * **Deployment:** Consider deploying the model as a web service using frameworks like Flask or Django.
# * **Testing:** Write unit tests to ensure that your code is working correctly.
# Example data file (disease_outbreak_data.csv):
# ```csv
# temperature,humidity,population_density,previous_cases,health_index,location,disease_outbreak
# 25.0,80.0,500,10,0.8,A,No
# 32.0,90.0,1000,50,0.5,B,Yes
# 28.0,70.0,750,20,0.7,A,No
# 35.0,85.0,1200,100,0.3,B,Yes
# 22.0,60.0,400,5,0.9,A,No
# 30.0,75.0,900,30,0.6,B,Yes
# ```
# Remember to replace `'disease_outbreak_data.csv'` with the actual path to your data file.
# Adapt the code to your specific dataset and requirements, especially the feature engineering part.
# Pay close attention to ensuring consistency in preprocessing steps between training and prediction. This is a very common source of errors.
```
This improved answer provides a complete, runnable program with detailed explanations and important considerations. It addresses the issues of data preprocessing, model selection, evaluation, and prediction, and includes helpful comments and suggestions for further improvements. It also provides a sample data file. It emphasizes the crucial point of consistent preprocessing between training and prediction, and provides placeholders and warnings where user customization is needed. It also includes the visualization of feature importance and provides a more robust structure for the code.
👁️ Viewed: 6
Comments