Regression Models and Neural Networks: A Comprehensive Guide

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate sample data
np.random.seed(42)
X = np.random.rand(100, 1) * 10
y = 3 * X + 2 + np.random.normal(0, 1, (100, 1))

# Fit linear regression
model = LinearRegression()
model.fit(X, y)

# Predictions
y_pred = model.predict(X)

# Model evaluation
mse = mean_squared_error(y, y_pred)
r2 = r2_score(y, y_pred)

print(f"Slope: {model.coef_[0][0]:.2f}")
print(f"Intercept: {model.intercept_[0]:.2f}")
print(f"R² Score: {r2:.3f}")
print(f"MSE: {mse:.3f}")

# Visualization
plt.figure(figsize=(10, 6))
plt.scatter(X, y, alpha=0.6, label='Data Points')
plt.plot(X, y_pred, color='red', linewidth=2, label='Regression Line')
plt.xlabel('Feature (X)')
plt.ylabel('Target (y)')
plt.title('Linear Regression')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

# Multiple features
X_multi = np.random.rand(100, 3) * 10
y_multi = (2 * X_multi[:, 0] + 1.5 * X_multi[:, 1] + 
           0.5 * X_multi[:, 2] + np.random.normal(0, 1, 100))

# Fit multiple linear regression
model_multi = LinearRegression()
model_multi.fit(X_multi, y_multi)

# Feature importance
feature_names = ['Feature_1', 'Feature_2', 'Feature_3']
for name, coef in zip(feature_names, model_multi.coef_):
    print(f"{name}: {coef:.3f}")

from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline

# Generate non-linear data
X_poly = np.random.rand(100, 1) * 4
y_poly = 0.5 * X_poly**3 - 2 * X_poly**2 + 3 * X_poly + np.random.normal(0, 0.5, (100, 1))

# Polynomial regression pipeline
poly_model = Pipeline([
    ('poly', PolynomialFeatures(degree=3)),
    ('linear', LinearRegression())
])

poly_model.fit(X_poly, y_poly)
y_poly_pred = poly_model.predict(X_poly)

# Visualization
plt.figure(figsize=(10, 6))
plt.scatter(X_poly, y_poly, alpha=0.6, label='Data Points')
plt.plot(X_poly, y_poly_pred, color='red', linewidth=2, label='Polynomial Fit')
plt.xlabel('Feature (X)')
plt.ylabel('Target (y)')
plt.title('Polynomial Regression (Degree 3)')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

from sklearn.linear_model import Ridge, Lasso
from sklearn.model_selection import train_test_split

# Split data
X_train, X_test, y_train, y_test = train_test_split(X_multi, y_multi, test_size=0.2, random_state=42)

# Ridge Regression (L2 regularization)
ridge_model = Ridge(alpha=1.0)
ridge_model.fit(X_train, y_train)
ridge_score = ridge_model.score(X_test, y_test)

# Lasso Regression (L1 regularization)
lasso_model = Lasso(alpha=0.1)
lasso_model.fit(X_train, y_train)
lasso_score = lasso_model.score(X_test, y_test)

print(f"Ridge R² Score: {ridge_score:.3f}")
print(f"Lasso R² Score: {lasso_score:.3f}")

# Compare coefficients
print("\nCoefficient Comparison:")
print("Feature\t\tLinear\t\tRidge\t\tLasso")
for i, name in enumerate(feature_names):
    print(f"{name}\t\t{model_multi.coef_[i]:.3f}\t\t{ridge_model.coef_[i]:.3f}\t\t{lasso_model.coef_[i]:.3f}")

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns

# Generate classification data
X_class = np.random.randn(200, 2)
y_class = (X_class[:, 0] + X_class[:, 1] > 0).astype(int)

# Fit logistic regression
log_model = LogisticRegression()
log_model.fit(X_class, y_class)

# Predictions
y_class_pred = log_model.predict(X_class)

# Evaluation
print("Classification Report:")
print(classification_report(y_class, y_class_pred))

# Confusion Matrix
cm = confusion_matrix(y_class, y_class_pred)
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam

# Prepare data
X_nn = np.random.rand(1000, 5) * 10
y_nn = (2 * X_nn[:, 0] + 1.5 * X_nn[:, 1] + 0.5 * X_nn[:, 2] + 
        0.3 * X_nn[:, 3] + 0.1 * X_nn[:, 4] + np.random.normal(0, 0.5, 1000))

# Split data
X_train, X_test, y_train, y_test = train_test_split(X_nn, y_nn, test_size=0.2, random_state=42)

# Build neural network
model_nn = Sequential([
    Dense(64, activation='relu', input_shape=(5,)),
    Dense(32, activation='relu'),
    Dense(16, activation='relu'),
    Dense(1, activation='linear')
])

# Compile model
model_nn.compile(optimizer=Adam(learning_rate=0.001),
                loss='mse',
                metrics=['mae'])

# Train model
history = model_nn.fit(X_train, y_train,
                      validation_split=0.2,
                      epochs=100,
                      batch_size=32,
                      verbose=0)

# Evaluate model
test_loss, test_mae = model_nn.evaluate(X_test, y_test, verbose=0)
print(f"Test MAE: {test_mae:.3f}")

# Plot training history
plt.figure(figsize=(12, 4))

plt.subplot(1, 2, 1)
plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history.history['mae'], label='Training MAE')
plt.plot(history.history['val_mae'], label='Validation MAE')
plt.title('Model MAE')
plt.xlabel('Epoch')
plt.ylabel('MAE')
plt.legend()

plt.tight_layout()
plt.show()

# More complex architecture
deep_model = Sequential([
    Dense(128, activation='relu', input_shape=(5,)),
    Dense(64, activation='relu'),
    Dense(32, activation='relu'),
    Dense(16, activation='relu'),
    Dense(8, activation='relu'),
    Dense(1, activation='linear')
])

# Add dropout for regularization
from tensorflow.keras.layers import Dropout

regularized_model = Sequential([
    Dense(128, activation='relu', input_shape=(5,)),
    Dropout(0.3),
    Dense(64, activation='relu'),
    Dropout(0.2),
    Dense(32, activation='relu'),
    Dropout(0.1),
    Dense(1, activation='linear')
])

# Compile with different optimizers
regularized_model.compile(optimizer='adam',
                         loss='mse',
                         metrics=['mae'])

# Custom loss function
def custom_loss(y_true, y_pred):
    mse = tf.keras.backend.mean(tf.keras.backend.square(y_true - y_pred))
    return mse + 0.01 * tf.keras.backend.mean(tf.keras.backend.abs(y_pred))

# Learning rate scheduling
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.001,
    decay_steps=1000,
    decay_rate=0.9
)

# Early stopping
early_stopping = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss',
    patience=10,
    restore_best_weights=True
)

# Model checkpoint
checkpoint = tf.keras.callbacks.ModelCheckpoint(
    'best_model.h5',
    monitor='val_loss',
    save_best_only=True
)

# Simulated house price data
np.random.seed(42)
n_samples = 1000

# Features: square_feet, bedrooms, bathrooms, age, location_score
house_features = np.random.rand(n_samples, 5)
house_features[:, 0] *= 3000  # Square feet: 0-3000
house_features[:, 1] = np.random.randint(1, 6, n_samples)  # Bedrooms: 1-5
house_features[:, 2] = np.random.randint(1, 4, n_samples)  # Bathrooms: 1-3
house_features[:, 3] = np.random.randint(0, 50, n_samples)  # Age: 0-50 years
house_features[:, 4] = np.random.rand(n_samples) * 10  # Location score: 0-10

# Target: house price (in thousands)
house_prices = (200 * house_features[:, 0] / 1000 +  # Base price per sq ft
                50 * house_features[:, 1] +           # Bedroom bonus
                30 * house_features[:, 2] +           # Bathroom bonus
                -2 * house_features[:, 3] +           # Age penalty
                25 * house_features[:, 4] +           # Location bonus
                np.random.normal(0, 20, n_samples))   # Noise

# Normalize features
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
house_features_scaled = scaler.fit_transform(house_features)

# Train neural network
house_model = Sequential([
    Dense(64, activation='relu', input_shape=(5,)),
    Dense(32, activation='relu'),
    Dense(16, activation='relu'),
    Dense(1, activation='linear')
])

house_model.compile(optimizer='adam', loss='mse', metrics=['mae'])

# Train
history = house_model.fit(house_features_scaled, house_prices,
                         validation_split=0.2,
                         epochs=50,
                         batch_size=32,
                         verbose=0)

# Predictions
predictions = house_model.predict(house_features_scaled)

# Visualization
plt.figure(figsize=(10, 6))
plt.scatter(house_prices, predictions, alpha=0.6)
plt.plot([house_prices.min(), house_prices.max()], 
         [house_prices.min(), house_prices.max()], 'r--', lw=2)
plt.xlabel('Actual Price (thousands)')
plt.ylabel('Predicted Price (thousands)')
plt.title('House Price Predictions')
plt.grid(True, alpha=0.3)
plt.show()

# Time series data for sales forecasting
import pandas as pd
from datetime import datetime, timedelta

# Generate time series data
dates = pd.date_range(start='2023-01-01', end='2024-12-31', freq='D')
n_days = len(dates)

# Features: day_of_week, month, season, holiday, marketing_spend
features = np.zeros((n_days, 5))
features[:, 0] = dates.dayofweek  # Day of week (0-6)
features[:, 1] = dates.month       # Month (1-12)
features[:, 2] = (dates.month % 12 + 3) // 3  # Season (1-4)
features[:, 3] = np.random.choice([0, 1], n_days, p=[0.9, 0.1])  # Holiday
features[:, 4] = np.random.exponential(1000, n_days)  # Marketing spend

# Sales with seasonality and trends
base_sales = 1000
seasonal_factor = 1 + 0.3 * np.sin(2 * np.pi * np.arange(n_days) / 365)
trend_factor = 1 + 0.001 * np.arange(n_days)
marketing_effect = 0.1 * features[:, 4] / 1000
holiday_boost = 0.5 * features[:, 3]

sales = (base_sales * seasonal_factor * trend_factor * 
         (1 + marketing_effect + holiday_boost) + 
         np.random.normal(0, 50, n_days))

# Create lagged features
def create_lagged_features(data, lags=[1, 7, 30]):
    lagged_data = data.copy()
    for lag in lags:
        lagged_data[f'lag_{lag}'] = data.shift(lag)
    return lagged_data.dropna()

# Prepare features for neural network
sales_df = pd.DataFrame({
    'sales': sales,
    'day_of_week': features[:, 0],
    'month': features[:, 1],
    'season': features[:, 2],
    'holiday': features[:, 3],
    'marketing_spend': features[:, 4]
})

# Add lagged features
sales_df = create_lagged_features(sales_df['sales']).join(sales_df.iloc[30:, 1:])

# Train neural network for time series
X_ts = sales_df.drop('sales', axis=1).values
y_ts = sales_df['sales'].values

# Split time series data
split_idx = int(0.8 * len(X_ts))
X_train_ts, X_test_ts = X_ts[:split_idx], X_ts[split_idx:]
y_train_ts, y_test_ts = y_ts[:split_idx], y_ts[split_idx:]

# Build time series model
ts_model = Sequential([
    Dense(64, activation='relu', input_shape=(X_train_ts.shape[1],)),
    Dense(32, activation='relu'),
    Dense(16, activation='relu'),
    Dense(1, activation='linear')
])

ts_model.compile(optimizer='adam', loss='mse', metrics=['mae'])

# Train
ts_history = ts_model.fit(X_train_ts, y_train_ts,
                         validation_split=0.2,
                         epochs=50,
                         batch_size=32,
                         verbose=0)

# Predictions
ts_predictions = ts_model.predict(X_test_ts)

# Plot results
plt.figure(figsize=(15, 5))
plt.plot(y_test_ts, label='Actual Sales', alpha=0.7)
plt.plot(ts_predictions, label='Predicted Sales', alpha=0.7)
plt.title('Sales Forecasting with Neural Network')
plt.xlabel('Time')
plt.ylabel('Sales')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

def evaluate_models(models, X_test, y_test):
    results = {}
    
    for name, model in models.items():
        if hasattr(model, 'predict'):
            y_pred = model.predict(X_test)
        else:
            y_pred = model(X_test, training=False).numpy().flatten()
        
        mae = mean_absolute_error(y_test, y_pred)
        mse = mean_squared_error(y_test, y_pred)
        rmse = np.sqrt(mse)
        r2 = r2_score(y_test, y_pred)
        
        results[name] = {
            'MAE': mae,
            'MSE': mse,
            'RMSE': rmse,
            'R²': r2
        }
    
    return results

# Compare different models
models = {
    'Linear Regression': model,
    'Polynomial Regression': poly_model,
    'Neural Network': model_nn
}

comparison_results = evaluate_models(models, X_test, y_test)

# Display results
print("Model Comparison:")
print("-" * 60)
for model_name, metrics in comparison_results.items():
    print(f"\n{model_name}:")
    for metric, value in metrics.items():
        print(f"  {metric}: {value:.3f}")

# Handle missing values
def handle_missing_values(df):
    # For numerical columns, fill with mean
    numerical_cols = df.select_dtypes(include=[np.number]).columns
    df[numerical_cols] = df[numerical_cols].fillna(df[numerical_cols].mean())
    
    # For categorical columns, fill with mode
    categorical_cols = df.select_dtypes(include=['object']).columns
    for col in categorical_cols:
        df[col] = df[col].fillna(df[col].mode()[0])
    
    return df

# Feature scaling
from sklearn.preprocessing import StandardScaler, MinMaxScaler

# StandardScaler for neural networks
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# MinMaxScaler for bounded outputs
minmax_scaler = MinMaxScaler()
y_scaled = minmax_scaler.fit_transform(y.reshape(-1, 1))

from sklearn.model_selection import cross_val_score, KFold

# K-fold cross-validation
kfold = KFold(n_splits=5, shuffle=True, random_state=42)

# For traditional models
cv_scores = cross_val_score(model, X, y, cv=kfold, scoring='r2')
print(f"Cross-validation R² scores: {cv_scores}")
print(f"Mean CV R²: {cv_scores.mean():.3f} (+/- {cv_scores.std() * 2:.3f})")

# For neural networks
def cross_validate_nn(X, y, n_splits=5):
    kfold = KFold(n_splits=n_splits, shuffle=True, random_state=42)
    scores = []
    
    for train_idx, val_idx in kfold.split(X):
        X_train_cv, X_val_cv = X[train_idx], X[val_idx]
        y_train_cv, y_val_cv = y[train_idx], y[val_idx]
        
        # Build and train model
        model_cv = Sequential([
            Dense(32, activation='relu', input_shape=(X.shape[1],)),
            Dense(16, activation='relu'),
            Dense(1, activation='linear')
        ])
        
        model_cv.compile(optimizer='adam', loss='mse', metrics=['mae'])
        model_cv.fit(X_train_cv, y_train_cv, epochs=50, verbose=0)
        
        # Evaluate
        score = model_cv.evaluate(X_val_cv, y_val_cv, verbose=0)[1]  # MAE
        scores.append(score)
    
    return np.array(scores)

from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestRegressor

# For traditional models
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [10, 20, None],
    'min_samples_split': [2, 5, 10]
}

rf = RandomForestRegressor(random_state=42)
grid_search = GridSearchCV(rf, param_grid, cv=5, scoring='r2', n_jobs=-1)
grid_search.fit(X_train, y_train)

print(f"Best parameters: {grid_search.best_params_}")
print(f"Best score: {grid_search.best_score_:.3f}")

# For neural networks (using Keras Tuner)
import keras_tuner as kt

def build_model(hp):
    model = Sequential()
    
    # Tune number of layers
    for i in range(hp.Int('num_layers', 1, 4)):
        model.add(Dense(
            units=hp.Int(f'units_{i}', 16, 128, step=16),
            activation='relu'
        ))
        if hp.Boolean('dropout'):
            model.add(Dropout(0.3))
    
    model.add(Dense(1, activation='linear'))
    
    model.compile(
        optimizer=hp.Choice('optimizer', ['adam', 'sgd']),
        loss='mse',
        metrics=['mae']
    )
    
    return model

# Initialize tuner
tuner = kt.Hyperband(
    build_model,
    objective='val_mae',
    max_epochs=50,
    factor=3,
    directory='hyperparameter_tuning',
    project_name='regression_nn'
)

# Search for best hyperparameters
tuner.search(X_train, y_train, validation_split=0.2, epochs=50)
best_model = tuner.get_best_models(1)[0]