Scripting Patterns & Environment Setup

# PyTorch implementation
from transformers import AutoModelForCausalLM, AutoTokenizer

model_name = "mistralai/Mistral-7B-Instruct-v0.3"

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype="auto",         # automatically choose float16/bfloat16
    device_map="auto",           # spread across available GPUs
    load_in_4bit=True,           # 4-bit quantization to save memory
)

# Most modern models use chat templates
messages = [
    {"role": "system", "content": "You are a helpful coding assistant."},
    {"role": "user", "content": "Write a Python function to compute Fibonacci numbers."},
]

# Apply the model's chat template
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

# Generate
outputs = model.generate(**inputs, max_new_tokens=256, temperature=0.7, do_sample=True)
response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print(response)

# implement tokenize_fn
from torch.utils.data import DataLoader
from datasets import load_dataset

# Load a dataset
dataset = load_dataset("imdb", split="test")

# Tokenize in batches
def tokenize_fn(examples):
    return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=512)

tokenized = dataset.map(tokenize_fn, batched=True, batch_size=64)
tokenized.set_format("torch", columns=["input_ids", "attention_mask", "label"])

# Create a DataLoader for efficient batched iteration
loader = DataLoader(tokenized, batch_size=16, shuffle=False)

# implement call_with_retry
import time
from openai import OpenAI

client = OpenAI()

def call_with_retry(prompt, max_retries=3, base_delay=1.0):
    """Call the API with exponential backoff on rate limits."""
    for attempt in range(max_retries):
        try:
            response = client.chat.completions.create(
                model="gpt-4o",
                messages=[{"role": "user", "content": prompt}],
                temperature=0.7,
            )
            return response.choices[0].message.content
        except Exception as e:
            if "rate_limit" in str(e).lower() and attempt < max_retries - 1:
                delay = base_delay * (2 ** attempt)
                print(f"Rate limited. Waiting {delay}s...")
                time.sleep(delay)
            else:
                raise

from transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
# Save a fine-tuned model
model.save_pretrained("./my-finetuned-model")
tokenizer.save_pretrained("./my-finetuned-model")
# Load it back later
model = AutoModelForCausalLM.from_pretrained("./my-finetuned-model")
tokenizer = AutoTokenizer.from_pretrained("./my-finetuned-model")
# Push to Hugging Face Hub
model.push_to_hub("your-username/my-finetuned-model")
tokenizer.push_to_hub("your-username/my-finetuned-model")

# Save a fine-tuned model and tokenizer to a local directory,
# then reload them and push to the Hugging Face Hub for sharing.
# Requires: pip install transformers huggingface-hub
from transformers import AutoModelForCausalLM, AutoTokenizer

# Save both model weights and tokenizer files to a local directory
model.save_pretrained("./my-finetuned-model")
tokenizer.save_pretrained("./my-finetuned-model")

# Reload the checkpoint later in any session
model = AutoModelForCausalLM.from_pretrained("./my-finetuned-model")
tokenizer = AutoTokenizer.from_pretrained("./my-finetuned-model")

# Push to the Hub so others can use it with from_pretrained("your-username/...")
model.push_to_hub("your-username/my-finetuned-model")
tokenizer.push_to_hub("your-username/my-finetuned-model")

# Activate the environment created per H.2, then install GPU-aware PyTorch.
# The pytorch-cuda meta-package pins the CUDA runtime to a matching wheel.
conda install pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch -c nvidia

# In your activated venv (set up per H.2):
# CUDA 12.4 wheels. Check https://pytorch.org/get-started/locally/ for current versions.
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124

# Verify the GPU build is active and CUDA is visible.
python -c "import torch; print(torch.cuda.is_available(), torch.version.cuda)"

# Core Hugging Face ecosystem
pip install transformers datasets tokenizers accelerate

# Fine-tuning and alignment
pip install peft trl bitsandbytes

# Inference serving
pip install vllm

# RAG and embeddings
pip install sentence-transformers langchain langchain-openai langchain-community langchain-text-splitters chromadb

# Experiment tracking
pip install wandb mlflow

# Data processing
pip install pandas numpy scikit-learn

# Jupyter notebooks
pip install jupyterlab ipywidgets

# Install vLLM (Linux only, or WSL2 on Windows)
pip install vllm

# Start a local inference server
python -m vllm.entrypoints.openai.api_server \
    --model meta-llama/Llama-3.1-8B-Instruct \
    --dtype auto \
    --max-model-len 4096

# Install vLLM (Linux or WSL2 only; requires a CUDA-compatible GPU).
# vLLM starts an OpenAI-compatible REST API server for local inference.
pip install vllm

# Launch the server; it will download the model weights on first run
python -m vllm.entrypoints.openai.api_server \
    --model meta-llama/Llama-3.1-8B-Instruct \
    --dtype auto \
    --max-model-len 4096

"""
LLM Environment Verification Script
Run this to confirm your setup is ready for the textbook exercises.
"""
import sys

def check_python():
    v = sys.version_info
    assert v.major == 3 and v.minor >= 10, f"Need Python 3.10+, got {v.major}.{v.minor}"
    print(f"[OK] Python {v.major}.{v.minor}.{v.micro}")

def check_torch():
    import torch
    print(f"[OK] PyTorch {torch.__version__}")
    if torch.cuda.is_available():
        print(f"[OK] CUDA {torch.version.cuda}, GPU: {torch.cuda.get_device_name(0)}")
        vram = torch.cuda.get_device_properties(0).total_mem / 1e9
        print(f"     VRAM: {vram:.1f} GB")
    else:
        print("[WARN] No CUDA GPU detected. CPU-only mode.")

def check_transformers():
    import transformers
    print(f"[OK] Transformers {transformers.__version__}")

def check_optional(name):
    try:
        mod = __import__(name)
        version = getattr(mod, "__version__", "installed")
        print(f"[OK] {name} {version}")
    except ImportError:
        print(f"[SKIP] {name} not installed (optional)")

def test_inference():
    """Quick smoke test: load a tiny model and generate one token."""
    from transformers import pipeline
    gen = pipeline("text-generation", model="sshleifer/tiny-gpt2", device="cpu")
    output = gen("Hello", max_new_tokens=5)
    assert len(output[0]["generated_text"]) > 0
    print("[OK] Inference smoke test passed")

if __name__ == "__main__":
    print("=== LLM Environment Check ===")
    check_python()
    check_torch()
    check_transformers()

    for lib in ["datasets", "peft", "trl", "bitsandbytes", "wandb", "vllm"]:
        check_optional(lib)

    print("\n=== Smoke Test ===")
    test_inference()

    print("\n=== All checks complete! ===")

# Experiment tracking setup
=== LLM Environment Check ===
[OK] Python 3.11.7
[OK] PyTorch 2.5.1
[OK] CUDA 12.4, GPU: NVIDIA GeForce RTX 4090
     VRAM: 24.0 GB
[OK] Transformers 4.47.0
[OK] datasets 3.3.0
[OK] peft 0.14.0
[OK] trl 0.13.0
[OK] bitsandbytes 0.45.0
[OK] wandb 0.19.1
[SKIP] vllm not installed (optional)

=== Smoke Test ===
[OK] Inference smoke test passed

=== All checks complete!

Python: 3.11.7
PyTorch: 2.4.0+cu124
CUDA available: True
CUDA device: NVIDIA RTX 4090 (24.0 GB)
Transformers: 4.45.2
Datasets: 3.0.1
Accelerate: 1.0.1
PEFT: 0.13.0

# Configure Git LFS to track large model files.
# LFS replaces large binaries with lightweight pointers in your repo and
# stores the actual blobs in remote LFS storage.
git lfs install

# Track common ML weight formats. Edit .gitattributes after running these.
git lfs track "*.safetensors"
git lfs track "*.bin"
git lfs track "*.pt"
git lfs track "*.ckpt"

# Commit the .gitattributes config so collaborators inherit it
git add .gitattributes
git commit -m "Track large model files with Git LFS"

# Normal workflow continues; LFS handles the size-aware upload/download
git add my-finetuned-model/model.safetensors
git commit -m "Add fine-tuned model"
git push