2025-02-08 · 3 min read

Auditing Dataset Quality Before Training

An audit catches dataset problems before you invest compute hours. The good ones are quick and signal obvious issues.

Three levels of validation

Format validation: JSON schema, required fields, template test.

Statistical audit: Token length distribution, class balance, duplication rate.

Human review sample: 20–30 random examples, sanity check on quality.

Format validation

import json
from datasets import load_dataset

def validate_format(jsonl_path, required_fields=None):
    """Check JSONL is valid JSON and has required fields."""
    if required_fields is None:
        required_fields = ["output"]

    errors = []
    for i, line in enumerate(open(jsonl_path)):
        try:
            obj = json.loads(line)
            for field in required_fields:
                if field not in obj:
                    errors.append(f"Line {i}: missing '{field}'")
        except json.JSONDecodeError as e:
            errors.append(f"Line {i}: invalid JSON: {e}")

    if errors:
        print(f"Format errors (first 10):")
        for err in errors[:10]:
            print(f"  {err}")
        return False

    print(f"✓ Format validation passed ({i+1} records)")
    return True

validate_format("data.jsonl", required_fields=["instruction", "output"])

Statistical audit

from transformers import AutoTokenizer
import json
import numpy as np

def audit_dataset(jsonl_path, model_name="Qwen/Qwen2.5-7B-Instruct"):
    """Compute token distribution and summary stats."""
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    records = [json.loads(line) for line in open(jsonl_path)]
    lengths = []

    for record in records:
        text = record.get("output", "")
        tokens = len(tokenizer.encode(text))
        lengths.append(tokens)

    lengths = np.array(lengths)

    print("Token length statistics:")
    print(f"  Count: {len(lengths)}")
    print(f"  Mean: {lengths.mean():.0f}")
    print(f"  Median (p50): {np.percentile(lengths, 50):.0f}")
    print(f"  p95: {np.percentile(lengths, 95):.0f}")
    print(f"  p99: {np.percentile(lengths, 99):.0f}")
    print(f"  Max: {lengths.max()}")

    # Warn about outliers
    p99_val = np.percentile(lengths, 99)
    outliers = sum(lengths > p99_val * 1.5)
    if outliers > len(lengths) * 0.05:
        print(f"  ⚠ {outliers} examples ({100*outliers/len(lengths):.1f}%) are extreme outliers")

    # Check for duplicates
    seen = set()
    dupes = 0
    for record in records:
        key = record.get("output", "")
        if key in seen:
            dupes += 1
        seen.add(key)

    if dupes > 0:
        print(f"  ⚠ {dupes} duplicate outputs detected ({100*dupes/len(records):.1f}%)")

    return {
        "count": len(lengths),
        "mean_tokens": lengths.mean(),
        "max_tokens": lengths.max(),
        "duplicates": dupes,
    }

audit_dataset("data.jsonl")

Example output:

Token length statistics:
  Count: 450
  Mean: 125
  Median (p50): 110
  p95: 280
  p99: 450
  Max: 2847
  ⚠ 8 examples (1.8%) are extreme outliers

Human review sample

import json
import random

def sample_for_review(jsonl_path, n=20):
    """Draw n random examples for manual inspection."""
    records = [json.loads(line) for line in open(jsonl_path)]
    sample = random.sample(records, min(n, len(records)))

    for i, record in enumerate(sample, 1):
        print(f"\n=== Sample {i} ===")
        print(f"Input: {record.get('instruction', '')}")
        print(f"Output: {record.get('output', '')[:200]}...")

sample_for_review("data.jsonl", n=20)

Red flags and fixes

PII check

import re
import json

def find_pii(jsonl_path):
    """Detect email, phone, names (basic)."""
    email_pattern = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
    phone_pattern = r'\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b'

    issues = []
    for i, line in enumerate(open(jsonl_path)):
        record = json.loads(line)
        text = record.get("output", "") + " " + record.get("instruction", "")

        if re.search(email_pattern, text):
            issues.append(f"Line {i}: email detected")
        if re.search(phone_pattern, text):
            issues.append(f"Line {i}: phone number detected")

    if issues:
        print(f"⚠ PII detected ({len(issues)} records):")
        for issue in issues[:5]:
            print(f"  {issue}")
    else:
        print("✓ No obvious PII found")

find_pii("data.jsonl")

Complete audit script

python audit.py \
  --input data.jsonl \
  --model Qwen/Qwen2.5-7B-Instruct \
  --sample-size 20

The script returns a JSON summary:

{
  "format_valid": true,
  "total_records": 450,
  "mean_tokens": 125,
  "max_tokens": 2847,
  "duplicates": 0,
  "pii_detected": false,
  "outliers": 8,
  "warnings": []
}

When to stop auditing

Audit stops when: - Format is valid - No obvious PII - Length distribution is reasonable (no 90% of examples being outliers) - Duplicate rate < 5%

Do not spend days perfecting the dataset before a single training run. Train, evaluate, then decide if more data prep is needed.