Advanced Feature Engineering Techniques
# Advanced Feature Engineering Techniques
## Introduction & Motivation
Feature engineering transforms raw data into informative representations, directly impacting ML model performance. Advanced techniques combine domain knowledge with automated discovery to create powerful features for complex engineering and scientific problems, improving accuracy and interpretability.
Motivation: Engineer features for superior ML model performance.
Applications: Predictive modeling, classification tasks, regression problems, pattern recognition.
---
## Core Concepts & Theory
### Domain-Specific Features
Physics-informed feature design.
### Automated Feature Discovery
Genetic algorithms and symbolic regression.
### Interaction Features
Capturing feature relationships.
### Feature Selection
Identifying high-value features.
---
## Mathematical Formulation
Polynomial Features:
$$\mathbf{x}' = [x_1, x_2, \ldots, x_1^2, x_1x_2, \ldots, x_2^2]$$
Basis Expansion:
$$\phi(\mathbf{x}) = [\phi_1(\mathbf{x}), \phi_2(\mathbf{x}), \ldots, \phi_M(\mathbf{x})]$$
Feature Importance:
$$I_i = \frac{1}{N} \sum_{t=1}^{T} \Delta MSE_i(t)$$
---
## Advanced Theory & Extensions
### Feature Interactions
Capturing non-linear relationships.
### Temporal Features
Time-based transformations.
### Dimensionality Reduction
PCA and manifold methods.
---
## Computational Considerations
Polynomial Features: O(D^k) for D features, k degree.
Feature Selection: O(2^D) exhaustive search.
Automated Discovery: O(G·P·T) for G generations, P population, T time.
---
## Practical Implementation Strategies
### Domain Knowledge Integration
Physics constraints in features.
### Statistical Methods
Correlation and mutual information.
### Machine Learning Approaches
Permutation importance and SHAP.
---
## Benchmark Datasets & Evaluation
UCI Repository: Benchmark datasets.
Kaggle Competitions: Real-world problems.
Domain Datasets: Engineering measurements.
---
## Key Challenges & Limitations
### Curse of Dimensionality
Too many features.
### Overfitting
Features capturing noise.
### Interpretability
Understanding feature impact.
---
## Hyperparameter Tuning
Polynomial degree: 2-4.
Feature selection threshold: 0.01-0.1.
Regularization: 0.0001-0.1.
---
## Real-World Applications & Case Studies
Sensor Data: Derived physics quantities.
Chemical: Molecular descriptors.
Materials: Structural representations.
---
## Integration with Other Methods
Feature engineering + model selection; + regularization; + validation.
---
## Summary & Key Takeaways
Advanced feature engineering drives ML model performance.
Principles:
1. Domain: Leverage domain knowledge.
2. Exploration: Discover informative features.
3. Selection: Choose high-impact features.
4. Validation: Rigorous feature assessment.
5. Interpretation: Understand feature roles.
---
## Appendix: Practical Labs
### Lab 1: Polynomial Feature Expansion
import numpy as np
class PolynomialFeatureExpander:
def __init__(self, degree=2, include_bias=True):
self.degree = degree
self.include_bias = include_bias
self.feature_names = None
def expand(self, X):
"""Expand to polynomial features"""
n_features = X.shape[1]
# Start with original features
X_expanded = X.copy()
# Add polynomial terms
for d in range(2, self.degree + 1):
X_powers = X ** d
X_expanded = np.column_stack([X_expanded, X_powers])
# Add interaction terms
if self.degree >= 2:
for i in range(n_features):
for j in range(i+1, n_features):
interactions = X[:, i] * X[:, j]
X_expanded = np.column_stack([X_expanded, interactions])
# Add bias
if self.include_bias:
X_expanded = np.column_stack([np.ones(len(X)), X_expanded])
return X_expanded
expander = PolynomialFeatureExpander(degree=2)
X = np.array([[1, 2], [3, 4], [5, 6]])
X_expanded = expander.expand(X)
print(f"✓ Polynomial feature expansion:")
print(f" Original shape: {X.shape}")
print(f" Expanded shape: {X_expanded.shape}")### Lab 2: Feature Importance
import numpy as np
def compute_feature_importance_permutation(model, X, y):
"""Compute importance via permutation"""
baseline_score = np.mean((model(X) - y) ** 2)
importances = np.zeros(X.shape[1])
for i in range(X.shape[1]):
X_shuffled = X.copy()
X_shuffled[:, i] = np.random.permutation(X_shuffled[:, i])
shuffled_score = np.mean((model(X_shuffled) - y) ** 2)
importances[i] = shuffled_score - baseline_score
return importances
def compute_correlation_importance(X, y):
"""Feature importance from correlation"""
correlations = np.array([np.abs(np.corrcoef(X[:, i], y)[0, 1]) for i in range(X.shape[1])])
# Handle NaN
correlations = np.nan_to_num(correlations)
return correlations / np.sum(correlations)
# Test
X = np.random.randn(100, 10)
y = X[:, 0] * 2 + X[:, 1] + np.random.randn(100) * 0.5
model = lambda x: x[:, :2] @ np.array([2, 1])
importance_perm = compute_feature_importance_permutation(model, X, y)
importance_corr = compute_correlation_importance(X, y)
print(f"✓ Feature importance (top 3):")
top_indices_perm = np.argsort(importance_perm)[-3:]
print(f" Permutation: {top_indices_perm}")
top_indices_corr = np.argsort(importance_corr)[-3:]
print(f" Correlation: {top_indices_corr}")### Lab 3: Automated Feature Discovery
import numpy as np
class SymbolicFeatureGenerator:
def __init__(self, feature_names=None):
self.feature_names = feature_names or [f'x{i}' for i in range(10)]
def generate_features(self, X, operations=['sqrt', 'log', 'square', 'inverse']):
"""Generate candidate features"""
features = []
for i, col in enumerate(X.T):
# Original
features.append((f'x{i}', col))
# Derived
if 'sqrt' in operations and np.all(col >= 0):
features.append((f'sqrt(x{i})', np.sqrt(col)))
if 'log' in operations and np.all(col > 0):
features.append((f'log(x{i})', np.log(col)))
if 'square' in operations:
features.append((f'x{i}^2', col ** 2))
if 'inverse' in operations and np.all(col != 0):
features.append((f'1/x{i}', 1.0 / col))
return features
def rank_features(self, X, y, features):
"""Rank features by correlation with target"""
scores = []
for name, feature_values in features:
# Handle NaN/Inf
if np.any(np.isnan(feature_values)) or np.any(np.isinf(feature_values)):
score = 0
else:
correlation = np.abs(np.corrcoef(feature_values, y)[0, 1])
score = np.nan_to_num(correlation)
scores.append((name, score))
# Sort by score
scores.sort(key=lambda x: x[1], reverse=True)
return scores
generator = SymbolicFeatureGenerator()
X = np.random.randn(50, 5)
y = np.sqrt(np.abs(X[:, 0])) + X[:, 1] ** 2 + np.random.randn(50) * 0.1
features = generator.generate_features(X)
ranked = generator.rank_features(X, y, features)
print(f"✓ Top 5 generated features:")
for name, score in ranked[:5]:
print(f" {name}: {score:.3f}")### Lab 4: Integrated Feature Engineering System
import numpy as np
class FeatureEngineeringPipeline:
def __init__(self, input_dim=10):
self.input_dim = input_dim
self.feature_transforms = []
self.feature_selection_indices = None
def add_polynomial_features(self, degree=2):
"""Add polynomial features"""
def poly_transform(X):
X_poly = X.copy()
for d in range(2, degree + 1):
X_poly = np.column_stack([X_poly, X ** d])
# Interactions
n = X.shape[1]
for i in range(min(3, n)):
for j in range(i+1, min(5, n)):
X_poly = np.column_stack([X_poly, X[:, i] * X[:, j]])
return X_poly
self.feature_transforms.append(('polynomial', poly_transform))
def add_statistical_features(self):
"""Add statistical features"""
def stat_transform(X):
stats = np.column_stack([
np.mean(X, axis=1),
np.std(X, axis=1),
np.max(X, axis=1),
np.min(X, axis=1)
])
return np.column_stack([X, stats])
self.feature_transforms.append(('statistics', stat_transform))
def select_features(self, X, y, n_features=10):
"""Select top features by correlation"""
X_engineered = X.copy()
# Apply all transforms
for name, transform in self.feature_transforms:
X_engineered = transform(X_engineered)
# Compute importances
importances = np.array([
np.abs(np.corrcoef(X_engineered[:, i], y)[0, 1])
if np.std(X_engineered[:, i]) > 0 else 0
for i in range(X_engineered.shape[1])
])
importances = np.nan_to_num(importances)
# Select top features
self.feature_selection_indices = np.argsort(importances)[-n_features:]
return X_engineered[:, self.feature_selection_indices]
def transform(self, X):
"""Apply full pipeline"""
X_result = X.copy()
for name, transform in self.feature_transforms:
X_result = transform(X_result)
if self.feature_selection_indices is not None:
X_result = X_result[:, self.feature_selection_indices]
return X_result
pipeline = FeatureEngineeringPipeline()
pipeline.add_polynomial_features(degree=2)
pipeline.add_statistical_features()
X_train = np.random.randn(100, 10)
y_train = np.sum(X_train[:, :3], axis=1) + np.random.randn(100) * 0.5
X_engineered = pipeline.select_features(X_train, y_train, n_features=15)
print(f"✓ Feature engineering pipeline:")
print(f" Original features: {X_train.shape[1]}")
print(f" Engineered features: {X_engineered.shape[1]}")---