Example: Research engineer debugging training instability
1. Investigate training loss spike at step 10,000
def investigate_loss_spike(checkpoint_dir, step_range):
    """Research engineers debug training issues through principled investigation"""
    # Load checkpoints before/during/after spike
    before_ckpt = load_checkpoint(f"{checkpoint_dir}/step_{step_range[0]}")
    during_ckpt = load_checkpoint(f"{checkpoint_dir}/step_{step_range[1]}")
    after_ckpt = load_checkpoint(f"{checkpoint_dir}/step_{step_range[2]}")
    # Analyze gradient statistics
    grad_stats = {
        'before': compute_grad_stats(before_ckpt),
        'during': compute_grad_stats(during_ckpt),
        'after': compute_grad_stats(after_ckpt),
    }
    # Check for gradient explosion, vanishing, or NaN
    if grad_stats['during']['max_grad'] > 1e6:
        print("Gradient explosion detected")
        # Investigate: learning rate too high? Bad batch? Numerical instability?
    # Inspect attention patterns
    attention_analysis = analyze_attention_patterns(during_ckpt)
    # Review training data from problematic batch
    problematic_batch = load_batch(step_range[1])
    data_quality_issues = check_data_quality(problematic_batch)
    return {
        'diagnosis': '...',
        'proposed_fix': '...',
        'experiment_plan': '...',
    }

Research Scientist <--- Research Engineer ---> ML Engineer
(Theory/Papers)        (Implementation)      (Production)

Core topics to master
ml_fundamentals = {
    'supervised_learning': [
        'Linear models, decision trees, neural networks',
        'Loss functions, optimization (SGD, Adam)',
        'Regularization, overfitting, generalization',
    ],
    'deep_learning': [
        'Backpropagation, computational graphs',
        'CNN architectures (ResNet, EfficientNet)',
        'RNN/LSTM, attention mechanisms',
        'Transformers in detail',
    ],
    'mathematics': [
        'Linear algebra: matrices, eigenvalues, SVD',
        'Calculus: gradients, chain rule, optimization',
        'Probability: distributions, Bayes theorem',
        'Information theory: entropy, KL divergence',
    ],
}
Study strategy
def study_plan_month_1_2():
    # Week 1-4: Fast.ai Practical Deep Learning
    complete_course("fast.ai")  # Top-down approach, get working models quickly
    # Week 5-8: DeepLearning.AI Specialization
    complete_course("deeplearning.ai")  # Bottom-up approach, understand fundamentals
    # Supplement with:
    read_textbook("Deep Learning" by Goodfellow)  # Chapters 1-12
    implement_from_scratch("Neural network with numpy")
    implement_from_scratch("Transformer from scratch")
    # Validation:
    can_explain("Why does batch normalization help training?")
    can_implement("Attention mechanism without looking at code")
    can_debug("Why is my model not converging?")

Research engineering requires systems knowledge
systems_skills = {
    'distributed_computing': [
        'Data parallelism, model parallelism',
        'AllReduce, gradient synchronization',
        'Distributed training frameworks (Horovod, DeepSpeed)',
    ],
    'gpu_programming': [
        'CUDA basics, kernel launches',
        'Memory hierarchy (global, shared, registers)',
        'Performance profiling with nsight',
    ],
    'ml_frameworks': [
        'PyTorch internals, autograd',
        'JAX and functional transformations',
        'Model serialization and checkpointing',
    ],
    'infrastructure': [
        'Docker containerization',
        'Kubernetes for ML workloads',
        'Cloud platforms (AWS, GCP, Azure)',
    ],
}
Project: Build a distributed training pipeline
def capstone_project():
    # Implement data parallelism for large model
    implement("Multi-GPU training with PyTorch DDP")
    optimize("Gradient accumulation for large batches")
    profile("Communication vs. computation bottlenecks")
    # Build monitoring and logging
    integrate("Weights & Biases for experiment tracking")
    implement("Custom metrics and visualizations")
    # Document everything
    write("Technical blog post on distributed training")

Choose a specialization area aligned with target labs
specializations = {
    'language_models': {
        'topics': ['Scaling laws', 'RLHF', 'Constitutional AI'],
        'projects': ['Fine-tune GPT-2', 'Implement RLHF from scratch'],
        'papers': ['InstructGPT', 'Constitutional AI', 'Chinchilla'],
    },
    'multimodal_models': {
        'topics': ['Vision transformers', 'CLIP', 'Diffusion models'],
        'projects': ['Train mini-CLIP', 'Implement stable diffusion'],
        'papers': ['CLIP', 'DALL-E', 'Stable Diffusion'],
    },
    'systems_optimization': {
        'topics': ['Quantization', 'Pruning', 'Knowledge distillation'],
        'projects': ['Optimize inference latency', 'Implement FlashAttention'],
        'papers': ['FlashAttention', 'Quantization papers', 'Distillation'],
    },
}
Build public portfolio
def build_portfolio():
    # GitHub projects showcasing skills
    create_repo("transformer-from-scratch")  # Well-documented implementation
    create_repo("distributed-training-template")  # Reusable training infrastructure
    create_repo("model-evaluation-framework")  # Comprehensive eval toolkit
    # Technical blog posts
    write_post("Understanding Self-Attention: A Visual Guide")
    write_post("Debugging Training Instabilities: A Systematic Approach")
    write_post("Scaling Transformers: From 1 GPU to 1000")
    # Open-source contributions (Critical!)
    contribute("PyTorch: Fix distributed training bug")
    contribute("Hugging Face Transformers: Add new model architecture")
    contribute("LlamaIndex: Improve documentation")

Dive deeper into frontier research areas
advanced_topics = {
    'training_dynamics': [
        'Scaling laws and compute-optimal training',
        'Learning rate schedules and warmup',
        'Batch size effects on convergence',
        'Loss landscape geometry',
    ],
    'alignment_and_safety': [
        'RLHF implementation details',
        'Constitutional AI and principles',
        'Red-teaming and adversarial testing',
        'Interpretability techniques',
    ],
    'efficiency': [
        'FlashAttention and fused kernels',
        'Quantization (GPTQ, QLoRA)',
        'Mixture of Experts architectures',
        'Sparse transformers',
    ],
}
Deep dive project
def advanced_project():
    # Reproduce a recent paper's results
    paper = "Constitutional AI from Human Feedback"
    # Implementation
    implement(paper)
    run_experiments("Replicate key results")
    analyze("Where results differ from paper")
    # Write-up
    publish("Reproduction study: What we learned")
    share("Code and trained models")

Strategic open-source contributions
contribution_strategy = {
    'target_projects': [
        # High-visibility projects where skills visible
        'ruff',  # Rust linter - demonstrates Rust proficiency
        'uv',    # Rust package manager - Python + Rust
        'pytorch',  # Core ML framework
        'transformers',  # Hugging Face
        'llama.cpp',  # LLM inference in C++
    ],
    'contribution_types': [
        'bug_fixes',  # Easier to get merged, build trust
        'performance_optimizations',  # Show systems skill
        'documentation',  # Understand codebase deeply
        'new_features',  # Most impactful, requires trust
    ],
    'goals': {
        'quantity': '15+ merged PRs',
        'quality': 'Meaningful contributions, not typo fixes',
        'visibility': 'Maintainers recognize your username',
    },
}
Max's approach: 15+ PRs in Ruff and UV
def replication():
    # 1. Choose project aligned with target lab's tech stack
    # Mistral uses Rust for inference -> contribute to Rust ML tools
    # 2. Start with good first issues
    find_issues("label:good-first-issue")
    solve("5-10 small issues to learn codebase")
    # 3. Tackle substantial issues
    identify("Performance bottleneck in hot path")
    profile("Measure current performance")
    optimize("Implement and benchmark improvement")
    submit("Well-documented PR with benchmarks")
    # 4. Build relationships with maintainers
    review("Others' PRs, provide thoughtful feedback")
    discuss("Architecture decisions in issues")
    propose("New features through RFC process")

Build professional network in AI community
networking_strategy = {
    'twitter': {
        'action': 'Share learning in public',
        'content': [
            'Paper summaries and insights',
            'Implementation challenges and solutions',
            'Performance optimization tricks',
            'Questions to spark discussions',
        ],
        'goal': 'Build reputation as someone who ships',
    },
    'conferences': {
        'attend': ['NeurIPS', 'ICML', 'ICLR'],  # If budget allows
        'alternative': 'Virtual attendance, follow Twitter discussions',
        'strategy': 'Connect with researchers, ask thoughtful questions',
    },
    'recurse_center': {
        'description': 'Self-directed learning retreat (3 months, free)',
        'benefits': [
            'Focused time for deep work',
            'Community of motivated learners',
            'Companies recruit from Recurse',
        ],
        'projects': 'Work on ambitious ML systems projects',
    },
}
Max's approach: Attended Recurse Center
def immersive_learning():
    # Apply to Recurse Center (or similar program)
    apply("Recurse Center")
    # If accepted, work on substantial project
    project = "Implement efficient transformer training in Rust"
    # Document journey publicly
    write("Weekly updates on progress")
    share("Code and learnings")
    # Network with batch-mates and alumni
    connect("Many work at target companies")
    seek("Referrals when ready to apply")

Research engineer interviews cover multiple domains
interview_prep = {
    'coding': {
        'platform': 'LeetCode, AlgoExpert',
        'focus': [
            'Medium-hard algorithm problems',
            'ML-specific algorithms (nearest neighbors, tree structures)',
            'Implement ML algorithms from scratch',
        ],
        'practice': '50+ problems, focus on ML-adjacent topics',
    },
    'ml_concepts': {
        'topics': [
            'Backpropagation and gradient computation',
            'Optimization algorithms and convergence',
            'Attention mechanisms and transformer architecture',
            'Training dynamics and instabilities',
            'Evaluation metrics and statistical significance',
        ],
        'preparation': 'Be able to explain and derive on whiteboard',
    },
    'systems_design': {
        'scenarios': [
            'Design distributed training system for 100B parameter model',
            'Build inference serving for low-latency API',
            'Implement evaluation pipeline for model capabilities',
        ],
        'skills': 'Trade-offs, bottlenecks, failure modes, monitoring',
    },
    'research_discussion': {
        'format': 'Discuss recent papers, your projects, research ideas',
        'preparation': [
            'Read 20+ recent papers deeply',
            'Have opinions on research directions',
            'Prepare thoughtful questions',
        ],
    },
}
Interview question examples
def example_questions():
    # Coding
    q1 = "Implement attention mechanism from scratch"
    q2 = "Write batched matrix multiplication with broadcasting"
    q3 = "Implement beam search for text generation"
    # ML Concepts
    q4 = "Explain why BatchNorm helps training"
    q5 = "Derive gradient for cross-entropy loss"
    q6 = "What causes training loss to spike mid-training?"
    # Systems
    q7 = "Design system to train 100B model on 1000 GPUs"
    q8 = "How would you debug 20% throughput drop in training?"
    q9 = "Optimize inference latency for transformer model"
    # Research
    q10 = "Discuss tradeoffs between different positional encodings"
    q11 = "How would you improve RLHF training stability?"
    q12 = "What's your take on mixture-of-experts scaling?"

Application strategy
application_plan = {
    'timeline': {
        'month_14': 'Apply to 10-15 companies',
        'month_15': 'Interview rounds',
        'month_16': 'Offers and decision',
    },
    'target_companies': [
        # Tier 1: Foundation model labs
        'OpenAI', 'Anthropic', 'Google DeepMind', 'Mistral', 'Cohere',
        # Tier 2: Strong AI teams
        'Meta FAIR', 'Microsoft Research', 'Inflection', 'Together.ai',
        # Tier 3: Strong ML companies
        'Hugging Face', 'Scale AI', 'Databricks', 'Weights & Biases',
    ],
    'referral_strategy': {
        'priority_1': 'Leverage open-source connections',
        'priority_2': 'Reach out to Recurse Center alumni',
        'priority_3': 'LinkedIn/Twitter connections',
        'priority_4': 'Cold applications with strong portfolio',
    },
}
Max's success: Leveraged Rust contributions and Recurse Center network
def max_approach():
    # Open-source visibility led to conversations
    result = "Maintainers familiar with work quality"
    # Recurse Center connections provided referrals
    result = "Multiple referrals from batch-mates and alumni"
    # Strong portfolio differentiated application
    result = "Tangible evidence of skills and shipping ability"
    # Outcome: Research engineer offer from Mistral
    return "Success after 18 months of focused effort"

Final months before offers
final_prep = {
    'research_deep_dive': {
        'action': 'Read and understand target lab's papers',
        'example': 'For Anthropic: Constitutional AI, Claude papers',
        'goal': 'Demonstrate deep familiarity with their work',
    },
    'technical_project': {
        'action': 'Build impressive project demonstrating research taste',
        'examples': [
            'Novel training technique with empirical validation',
            'Comprehensive evaluation framework for model capabilities',
            'Performance optimization with significant speedup',
        ],
    },
    'communication': {
        'action': 'Practice explaining technical concepts clearly',
        'formats': ['Whiteboard explanations', 'Code walkthroughs', 'Research discussions'],
    },
}

Must be proficient in scientific Python stack
essential_libraries = {
    'numpy': 'Array operations, broadcasting, vectorization',
    'pytorch': 'Autograd, distributed training, JIT compilation',
    'jax': 'Functional transformations, auto-vectorization',
    'pandas': 'Data manipulation and analysis',
    'matplotlib': 'Visualization for debugging and analysis',
}

Example: Implement a training loop from scratch
def training_loop(model, dataloader, optimizer, num_epochs):
    """Research engineers write training code from scratch often"""
    for epoch in range(num_epochs):
        for batch_idx, (data, targets) in enumerate(dataloader):
            # Forward pass
            predictions = model(data)
            loss = compute_loss(predictions, targets)
            # Backward pass
            optimizer.zero_grad()
            loss.backward()
            # Gradient clipping
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
            # Optimizer step
            optimizer.step()
            # Logging
            if batch_idx % 100 == 0:
                log_metrics(epoch, batch_idx, loss.item())
            # Checkpointing
            if batch_idx % 1000 == 0:
                save_checkpoint(model, optimizer, epoch, batch_idx)

// Rust increasingly important for ML systems
// Max's strategy: Learn Rust, contribute to Rust ML projects

// Example: Efficient tokenization in Rust
use rayon::prelude::*;
pub fn parallel_tokenize(texts: &[String], tokenizer: &Tokenizer) -> Vec> {
    texts.par_iter()
        .map(|text| tokenizer.encode(text))
        .collect()
}
// Why Rust matters for research engineering:
// - Performance-critical inference systems
// - Training infrastructure components
// - Data loading and preprocessing pipelines
// - Growing ecosystem (HuggingFace Tokenizers, SafeTensors, Candle)

// Useful for CUDA kernels and framework internals
// Example: Custom CUDA kernel
__global__ void flash_attention_kernel(
    const float* Q, const float* K, const float* V,
    float* output, int seq_len, int dim) {
    // Implement FlashAttention algorithm
    // Requires understanding GPU memory hierarchy
}

Research engineers must understand transformers intimately
class TransformerDeepDive:
    """Areas to master"""
    def attention_mechanism(self):
        """
        - Scaled dot-product attention derivation
        - Why scaling by sqrt(d_k)?
        - Multi-head attention purpose
        - Attention patterns and interpretability
        """
    def positional_encoding(self):
        """
        - Sinusoidal vs. learned positional embeddings
        - Absolute vs. relative position encodings
        - RoPE (Rotary Position Embeddings)
        - ALiBi (Attention with Linear Biases)
        """
    def training_dynamics(self):
        """
        - Warmup schedules and why they matter
        - Layer normalization placement (pre-norm vs. post-norm)
        - Gradient flow through deep transformers
        - Training instabilities and how to fix them
        """
    def efficiency(self):
        """
        - FlashAttention algorithm
        - KV caching for inference
        - Quantization effects on attention
        - Sparse attention patterns
        """

Understanding compute scaling is critical
scaling_laws = {
    'chinchilla_optimal': {
        'insight': 'Models were undertrained, data-constrained',
        'formula': 'N (params) ∝ D (data) for compute-optimal',
        'implication': 'Smaller, longer-trained models outperform larger, shorter-trained',
    },
    'emergent_abilities': {
        'observation': 'Capabilities emerge at specific scale thresholds',
        'examples': ['Few-shot learning', 'Chain-of-thought reasoning', 'Instruction following'],
        'research_question': 'Can we predict emergent capabilities?',
    },
    'compute_budget': {
        'training_cost': '$1M-$100M for foundation models',
        'optimization_goal': 'Maximize performance for given compute budget',
        'tradeoffs': 'Model size vs. training time vs. data quality',
    },
}

Critical skill for research engineers
distributed_concepts = {
    'data_parallelism': {
        'description': 'Replicate model, split data across GPUs',
        'implementation': 'PyTorch DistributedDataParallel (DDP)',
        'scaling': 'Linear speedup up to network bottleneck',
    },
    'model_parallelism': {
        'description': 'Split model across GPUs (for huge models)',
        'types': ['Tensor parallelism', 'Pipeline parallelism'],
        'frameworks': ['DeepSpeed', 'Megatron-LM', 'FairScale'],
    },
    'optimization': {
        'gradient_accumulation': 'Simulate larger batch sizes',
        'mixed_precision': 'FP16 training with loss scaling',
        'zero_optimization': 'DeepSpeed ZeRO for memory efficiency',
    },
}
Example: Implement gradient accumulation
def train_with_gradient_accumulation(
    model, dataloader, optimizer,
    accumulation_steps=4):
    """Accumulate gradients over multiple batches"""
    optimizer.zero_grad()
    for batch_idx, (data, targets) in enumerate(dataloader):
        # Forward pass
        predictions = model(data)
        loss = compute_loss(predictions, targets)
        # Normalize loss by accumulation steps
        loss = loss / accumulation_steps
        # Backward pass (accumulate gradients)
        loss.backward()
        # Update weights every accumulation_steps batches
        if (batch_idx + 1) % accumulation_steps == 0:
            optimizer.step()
            optimizer.zero_grad()

Understanding GPU performance is valuable
gpu_concepts = {
    'memory_hierarchy': {
        'global_memory': 'Large (40GB+), slow',
        'shared_memory': 'Small (96KB), fast, per-SM',
        'registers': 'Tiny, fastest, per-thread',
    },
    'performance_optimization': {
        'coalesced_access': 'Align memory access patterns',
        'occupancy': 'Balance threads vs. registers vs. shared memory',
        'kernel_fusion': 'Combine operations to reduce memory transfers',
    },
    'profiling': {
        'tools': ['nsight Compute', 'nsight Systems', 'PyTorch profiler'],
        'metrics': ['Kernel duration', 'Memory throughput', 'Occupancy'],
    },
}

labs = {
    'openai': {
        'focus': 'AGI, GPT series, multimodal models',
        'culture': 'Research-driven, ambitious goals',
        'compensation': 'Very high ($300K-$600K+)',
        'competition': 'Extremely competitive',
    },
    'anthropic': {
        'focus': 'AI safety, Constitutional AI, Claude',
        'culture': 'Safety-first, principled approach',
        'compensation': 'Very high ($300K-$600K+)',
        'competition': 'Extremely competitive',
    },
    'google_deepmind': {
        'focus': 'Research breadth, Gemini, AGI',
        'culture': 'Academic, publication-focused',
        'compensation': 'High ($250K-$500K)',
        'competition': 'Extremely competitive',
    },
    'mistral': {
        'focus': 'Open models, efficiency, European alternative',
        'culture': 'Fast-moving, European team',
        'compensation': 'High ($200K-$400K, equity upside)',
        'competition': 'Very competitive',
        'advantage': 'Smaller team, more impact',
    },
    'cohere': {
        'focus': 'Enterprise LLMs, RAG',
        'culture': 'Product-focused',
        'compensation': 'High ($200K-$400K)',
        'competition': 'Very competitive',
    },
}
Selection criteria
def choose_target_labs():
    considerations = {
        'research_alignment': 'What problems interest you?',
        'culture_fit': 'Academic vs. product-focused?',
        'geography': 'Willing to relocate?',
        'safety_focus': 'How important is AI safety to you?',
        'company_stage': 'Established vs. growing startup?',
    }
    # Max chose Mistral: growing team, Rust tech stack, European
    return "Choose 3-5 target labs aligned with your goals"

alternative_paths = {
    'ml_engineer_at_faang': {
        'description': 'Apply ML to products (recommendations, ads, etc.)',
        'compensation': '$250K-$500K at senior levels',
        'path': 'Easier to break into than research labs',
    },
    'ai_startup': {
        'description': 'Join early-stage AI company',
        'compensation': '$150K-$300K + significant equity',
        'risk': 'Higher risk, potential upside',
    },
    'ml_at_enterprise': {
        'description': 'Build ML systems for traditional companies',
        'compensation': '$200K-$400K',
        'lifestyle': 'Often better work-life balance',
    },
    'consulting': {
        'description': 'ML consulting for clients',
        'compensation': '$150K-$350K',
        'variety': 'Different projects, lots of variety',
    },
}

Framework for evaluating offers
offer_evaluation = {
    'compensation': {
        'base_salary': 'Immediate financial stability',
        'equity': 'Long-term upside potential',
        'sign_on_bonus': 'Short-term cash',
        'total_comp_4_year': 'Standard comparison metric',
    },
    'role_scope': {
        'autonomy': 'How much independence?',
        'impact': 'Working on core vs. auxiliary projects?',
        'growth': 'Learning opportunities?',
        'team': 'Quality of colleagues?',
    },
    'company_trajectory': {
        'funding': 'How much runway?',
        'product': 'Market fit? Competitive position?',
        'mission': 'Alignment with your values?',
    },
    'lifestyle': {
        'wlb': 'Expected hours? On-call?',
        'location': 'Office requirements? Remote options?',
        'culture': 'Will you enjoy working here?',
    },
}

How to compete without a PhD
strategies = {
    'demonstrate_research_ability': {
        'method': 'Reproduce papers, run experiments, publish findings',
        'evidence': 'Technical blog posts, open-source projects',
    },
    'show_implementation_excellence': {
        'method': 'Build high-quality ML systems, contribute to major projects',
        'evidence': 'GitHub PRs, production systems',
    },
    'signal_intellectual_curiosity': {
        'method': 'Read papers, engage in research discussions, propose ideas',
        'evidence': 'Twitter, blog posts, conference attendance',
    },
    'leverage_open_source': {
        'method': 'Become known contributor to ML projects',
        'evidence': 'Maintainer status, significant PRs',
    },
}
Max's success: No PhD, but demonstrated ability through:
- Rust contributions (rare, valuable skill)
- Recurse Center (immersive learning)
- Public learning and portfolio

18-month transition plan (condensed)
transition = {
    'months_1_6': {
        'focus': 'ML fundamentals and first projects',
        'outcome': 'Can train models, understand papers',
    },
    'months_7_12': {
        'focus': 'Specialization and portfolio building',
        'outcome': 'GitHub projects, blog posts, contributions',
    },
    'months_13_18': {
        'focus': 'Advanced topics and applications',
        'outcome': 'Ready for research engineering interviews',
    },
}
Leverage existing SWE skills
advantages = {
    'software_engineering': 'Production systems, code quality, debugging',
    'systems_knowledge': 'Distributed systems, performance optimization',
    'problem_solving': 'Debugging, systematic approaches',
}

Week 1 action plan
week_1 = {
    'day_1': 'Start Fast.ai course, lesson 1',
    'day_2': 'Set up ML development environment (PyTorch, Jupyter)',
    'day_3': 'Train your first neural network (MNIST)',
    'day_4': 'Read "Attention Is All You Need" paper (don't worry if confusing)',
    'day_5': 'Start following ML researchers on Twitter',
    'day_6': 'Join ML Discord communities (Hugging Face, EleutherAI)',
    'day_7': 'Write blog post: "Week 1 of my ML journey"',
}
The key: Start NOW, learn in public, be consistent