Original weight W (d x k)
         ↓
W' = W + BA  where B (d x r), A (r x k)
         ↓
Only B and A are trained (r << min(d, k))

from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is LoRA?"}, {"role": "assistant", "content": "LoRA (Low-Rank Adaptation) is..."}]}
{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "How to fine-tune?"}, {"role": "assistant", "content": "To fine-tune a model..."}]}

from openai import OpenAI

client = OpenAI()

# Upload training file
with open("training_data.jsonl", "rb") as f:
    file = client.files.create(file=f, purpose="fine-tune")

# Create fine-tuning job
job = client.fine_tuning.jobs.create(
    training_file=file.id,
    model="gpt-4o-mini-2024-07-18",
    hyperparameters={
        "n_epochs": 3,
        "batch_size": 4,
        "learning_rate_multiplier": 1.8
    },
    suffix="my-custom-model"
)

# Monitor progress
print(f"Job ID: {job.id}")
print(f"Status: {job.status}")

# List events
events = client.fine_tuning.jobs.list_events(job.id)
for event in events.data:
    print(event.message)

# Use fine-tuned model
response = client.chat.completions.create(
    model="ft:gpt-4o-mini-2024-07-18:my-org::abc123",
    messages=[{"role": "user", "content": "Hello!"}]
)

git clone https://github.com/hiyouga/LLaMA-Factory.git
cd LLaMA-Factory
pip install -e ".[torch,metrics]"

# Launch WebUI
python src/webui.py

# LoRA fine-tuning
llamafactory-cli train \
    --stage sft \
    --model_name_or_path meta-llama/Llama-3.1-8B \
    --dataset alpaca_en \
    --template llama3 \
    --finetuning_type lora \
    --lora_rank 16 \
    --lora_target q_proj,v_proj \
    --output_dir output/llama3-lora \
    --per_device_train_batch_size 2 \
    --gradient_accumulation_steps 4 \
    --learning_rate 5e-5 \
    --num_train_epochs 3 \
    --fp16

# Merge LoRA adapter
llamafactory-cli export \
    --model_name_or_path meta-llama/Llama-3.1-8B \
    --adapter_name_or_path output/llama3-lora \
    --template llama3 \
    --finetuning_type lora \
    --export_dir merged_model

[
  {
    "instruction": "Summarize the following text",
    "input": "Long text here...",
    "output": "Summary here..."
  }
]

[
  {
    "conversations": [
      {"from": "human", "value": "What is AI?"},
      {"from": "gpt", "value": "AI stands for..."}
    ]
  }
]

pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
pip install --no-deps trl peft accelerate bitsandbytes

from unsloth import FastLanguageModel
from trl import SFTTrainer
from transformers import TrainingArguments

# Load model (4-bit)
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Llama-3.2-1B",
    max_seq_length=2048,
    dtype=None,  # Auto-detect
    load_in_4bit=True,
)

# Add LoRA adapters
model = FastLanguageModel.get_peft_model(
    model,
    r=16,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    lora_alpha=16,
    lora_dropout=0,
    bias="none",
    use_gradient_checkpointing="unsloth",  # 30% less VRAM
    random_state=42,
)

# Training arguments
training_args = TrainingArguments(
    output_dir="./outputs",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    warmup_steps=10,
    max_steps=60,
    learning_rate=2e-4,
    fp16=not torch.cuda.is_bf16_supported(),
    bf16=torch.cuda.is_bf16_supported(),
    logging_steps=1,
    optim="adamw_8bit",
    weight_decay=0.01,
    lr_scheduler_type="linear",
    seed=42,
)

# Train
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
    args=training_args,
)

trainer.train()

# Save LoRA
model.save_pretrained("lora_model")
tokenizer.save_pretrained("lora_model")

# Save merged 16-bit
model.save_pretrained_merged("merged_16bit", tokenizer, save_method="merged_16bit")

# Save GGUF for llama.cpp / Ollama
model.save_pretrained_gguf("model_gguf", tokenizer, quantization_method="q4_k_m")

pip install axolotl
# Or from source
git clone https://github.com/OpenAccess-AI-Collective/axolotl
cd axolotl
pip install -e ".[flash-attn,deepspeed]"

# config.yaml
base_model: meta-llama/Llama-3.1-8B
model_type: LlamaForCausalLM
tokenizer_type: AutoTokenizer

load_in_8bit: false
load_in_4bit: true
strict: false

datasets:
  - path: my_dataset.jsonl
    type: alpaca
    data_files:
      - train.jsonl

dataset_prepared_path: prepared_data
val_set_size: 0.05
output_dir: ./outputs

adapter: lora
lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_modules:
  - q_proj
  - v_proj
  - k_proj
  - o_proj
  - gate_proj
  - down_proj
  - up_proj

sequence_len: 4096
sample_packing: true
pad_to_sequence_len: true

gradient_accumulation_steps: 4
micro_batch_size: 2
num_epochs: 3
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 2e-4
warmup_ratio: 0.03
weight_decay: 0.01

train_on_inputs: false
group_by_length: false
bf16: auto
fp16: false
tf32: false

gradient_checkpointing: true
flash_attention: true
deepspeed: null

wandb_project: my-finetune
wandb_run_id: run-001
wandb_log_model: false

special_tokens:
  pad_token: "<|end_of_text|>"

# Single GPU
accelerate launch -m axolotl.cli.train config.yaml

# Multi-GPU (DeepSpeed)
accelerate launch --config_file deepspeed.yaml -m axolotl.cli.train config.yaml

# Inference test
accelerate launch -m axolotl.cli.inference config.yaml --lora_model_dir ./outputs

import json
from datasets import load_dataset

def clean_dataset(raw_data):
    cleaned = []
    seen = set()

    for item in raw_data:
        # Remove duplicates
        key = (item.get("instruction", ""), item.get("output", ""))
        if key in seen:
            continue
        seen.add(key)

        # Filter empty
        if not item.get("output", "").strip():
            continue

        # Filter too short
        if len(item.get("output", "")) < 10:
            continue

        # Filter too long (adjust as needed)
        if len(item.get("output", "")) > 4096:
            continue

        cleaned.append(item)

    return cleaned

# Load and clean
with open("raw_data.json") as f:
    raw = json.load(f)

cleaned = clean_dataset(raw)
print(f"Cleaned: {len(raw)} -> {len(cleaned)}")

# Save
with open("cleaned_data.json", "w") as f:
    json.dump(cleaned, f, indent=2)

def format_alpaca(example):
    """Format for instruction-following."""
    if example.get("input"):
        return f"""### Instruction:
{example['instruction']}

### Input:
{example['input']}

### Response:
{example['output']}"""
    else:
        return f"""### Instruction:
{example['instruction']}

### Response:
{example['output']}"""

def format_chat(example):
    """Format for chat models."""
    messages = []
    for msg in example["conversations"]:
        role = "user" if msg["from"] == "human" else "assistant"
        messages.append({"role": role, "content": msg["value"]})
    return messages

# Effective batch size = batch_size * gradient_accumulation_steps * num_gpus
# Example: 2 * 4 * 1 = 8 effective batch size
training_args = TrainingArguments(
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
)

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

def calculate_perplexity(model, tokenizer, text, device="cuda"):
    encodings = tokenizer(text, return_tensors="pt").to(device)

    with torch.no_grad():
        outputs = model(**encodings, labels=encodings.input_ids)

    return torch.exp(outputs.loss).item()

# Usage
ppl = calculate_perplexity(model, tokenizer, "Sample text here")
print(f"Perplexity: {ppl:.2f}")

pip install lm-eval

# Evaluate on multiple benchmarks
lm_eval --model hf \
    --model_args pretrained=./my_model \
    --tasks mmlu,hellaswag,truthfulqa \
    --device cuda:0 \
    --batch_size 8

pip install mergekit

# Create merge config (merge.yaml)

# merge.yaml
slices:
  - sources:
      - model: base_model
        layer_range: [0, 32]
      - model: finetuned_model
        layer_range: [0, 32]
    merge_method: slerp
    base_model: base_model
    parameters:
      t: 0.5

merge_method: slerp
dtype: float16

# Run merge
mergekit-yaml merge.yaml ./merged_output

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

# Load base model
base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    torch_dtype=torch.float16,
    device_map="auto",
)

# Load LoRA
model = PeftModel.from_pretrained(base_model, "path/to/lora")

# Merge and unload
merged = model.merge_and_unload()

# Save
merged.save_pretrained("merged_model")

SFT Model → Reward Model → PPO Training → RLHF Model

from trl import DPOTrainer, DPOConfig

# Dataset format
# {"prompt": "...", "chosen": "preferred response", "rejected": "worse response"}

dpo_config = DPOConfig(
    beta=0.1,  # KL penalty
    learning_rate=5e-7,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=1,
)

trainer = DPOTrainer(
    model=model,
    ref_model=ref_model,
    args=dpo_config,
    train_dataset=dataset,
    tokenizer=tokenizer,
)

trainer.train()

from trl import ORPOTrainer, ORPOConfig

orpo_config = ORPOConfig(
    beta=0.1,
    learning_rate=8e-6,
    per_device_train_batch_size=2,
    num_train_epochs=1,
)

trainer = ORPOTrainer(
    model=model,
    args=orpo_config,
    train_dataset=dataset,
    tokenizer=tokenizer,
)

# Using Unsloth
model.save_pretrained_gguf("model", tokenizer, quantization_method="q4_k_m")

# Quantization options
# q4_k_m - Good balance (recommended)
# q5_k_m - Better quality
# q8_0 - Highest quality
# q2_k - Smallest size

# Modelfile
FROM ./model-q4_k_m.gguf

TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>

{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>

{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>

{{ .Response }}<|eot_id|>"""

PARAMETER stop "<|start_header_id|>"
PARAMETER stop "<|end_header_id|>"
PARAMETER stop "<|eot_id|>"

ollama create my-model -f Modelfile
ollama run my-model

pip install vllm

# Start server
python -m vllm.entrypoints.openai.api_server \
    --model ./merged_model \
    --port 8000 \
    --tensor-parallel-size 1

# API call
curl http://localhost:8000/v1/completions \
    -H "Content-Type: application/json" \
    -d '{"model": "merged_model", "prompt": "Hello", "max_tokens": 50}'

docker run --gpus all -p 8080:80 \
    -v $PWD/model:/data \
    ghcr.io/huggingface/text-generation-inference:latest \
    --model-id /data \
    --max-input-length 4096 \
    --max-total-tokens 8192

def estimate_training_time(
    dataset_size: int,
    batch_size: int,
    gradient_accumulation: int,
    epochs: int,
    tokens_per_second: int = 1000,
    avg_seq_length: int = 512
):
    """Estimate training time in hours."""
    steps = (dataset_size // (batch_size * gradient_accumulation)) * epochs
    total_tokens = steps * batch_size * gradient_accumulation * avg_seq_length
    hours = total_tokens / tokens_per_second / 3600
    return hours

# Example
hours = estimate_training_time(
    dataset_size=10000,
    batch_size=2,
    gradient_accumulation=4,
    epochs=3,
    tokens_per_second=2000,  # A100
)
print(f"Estimated: {hours:.1f} hours")

# Check GPU memory
import torch
print(f"Allocated: {torch.cuda.memory_allocated() / 1e9:.2f} GB")
print(f"Reserved: {torch.cuda.memory_reserved() / 1e9:.2f} GB")

# Monitor training
from transformers import TrainerCallback

class LogCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        if logs:
            print(f"Step {state.global_step}: loss={logs.get('loss', 'N/A'):.4f}")