JSONL (JSON Lines), also known as NDJSON, is a format where each line is a valid, self-contained JSON object. It is the de facto standard for LLM training data, application logs, and large-scale data pipelines.
JSONL vs JSON vs CSV
| Feature | JSON | JSONL | CSV |
|---|---|---|---|
| Streaming | No | Yes | Partial |
| Append without re-parsing | No | Yes | Yes |
| Nested data | Yes | Yes | No |
| ML/LLM use | No | Standard | No |
JSONL file example
{"id": 1, "name": "Alice", "tags": ["python", "data"]}
{"id": 2, "name": "Bob", "tags": ["web", "devops"]}
{"id": 3, "name": "Carol", "score": 9.5, "active": true}
Reading JSONL
import json
def read_jsonl(path):
with open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
yield json.loads(line)
for record in read_jsonl('data.jsonl'):
print(record['name'])
records = list(read_jsonl('data.jsonl'))
# Count without loading
with open('data.jsonl') as f:
total = sum(1 for l in f if l.strip())
Writing JSONL
import json
data = [{'id': 1, 'text': 'First', 'value': 42}, {'id': 2, 'text': 'Second', 'value': 87}]
with open('output.jsonl', 'w', encoding='utf-8') as f:
for record in data:
f.write(json.dumps(record, ensure_ascii=False) + '\n')
# Append
with open('output.jsonl', 'a', encoding='utf-8') as f:
f.write(json.dumps({'id': 3, 'text': 'New'}, ensure_ascii=False) + '\n')
Processing with pandas
import pandas as pd
df = pd.read_json('data.jsonl', lines=True)
print(df.head())
df_active = df[df['active'] == True]
df_active.to_json('active.jsonl', orient='records', lines=True, force_ascii=False)
# Read compressed directly
df_gz = pd.read_json('data.jsonl.gz', lines=True, compression='gzip')
Streaming pipeline for large files
import json
def jsonl_pipeline(input_path, output_path, filter_fn, transform_fn=None):
processed = written = 0
with open(input_path, 'r', encoding='utf-8') as fin, \
open(output_path, 'w', encoding='utf-8') as fout:
for line in fin:
line = line.strip()
if not line:
continue
processed += 1
record = json.loads(line)
if not filter_fn(record):
continue
if transform_fn:
record = transform_fn(record)
fout.write(json.dumps(record, ensure_ascii=False) + '\n')
written += 1
print(f"Processed: {processed} | Written: {written}")
jsonl_pipeline(
'sales.jsonl', 'large_sales.jsonl',
filter_fn=lambda r: r.get('amount', 0) > 1000,
transform_fn=lambda r: {**r, 'category': 'large'},
)
Convert JSON ↔ JSONL and compress
import json, gzip
# JSON array → JSONL
with open('list.json') as fin, open('list.jsonl', 'w') as fout:
for item in json.load(fin):
fout.write(json.dumps(item, ensure_ascii=False) + '\n')
# JSONL → JSON array
with open('list.jsonl') as fin, open('list.json', 'w') as fout:
records = [json.loads(l) for l in fin if l.strip()]
json.dump(records, fout, ensure_ascii=False, indent=2)
# Compress (reduces size 70-80%)
with open('data.jsonl', 'rb') as fin, gzip.open('data.jsonl.gz', 'wb') as fout:
fout.write(fin.read())
Validate JSONL
import json
def validate_jsonl(path, required_keys=None):
errors = []
total = 0
with open(path, 'r', encoding='utf-8') as f:
for num, line in enumerate(f, 1):
line = line.strip()
if not line:
continue
total += 1
try:
record = json.loads(line)
if required_keys:
missing = [k for k in required_keys if k not in record]
if missing:
errors.append(f"Line {num}: missing {missing}")
except json.JSONDecodeError as e:
errors.append(f"Line {num}: {e}")
print(f"Total: {total} | Errors: {len(errors)}")
return len(errors) == 0
validate_jsonl('data.jsonl', required_keys=['id', 'text'])
Main use cases
- LLM training data: OpenAI fine-tuning, Hugging Face datasets use JSONL as standard
- Application logs: one event per line, easily processed with grep/jq
- Database exports: MongoDB, Elasticsearch export in JSONL by default
- ETL pipelines: incremental processing of millions of records without RAM constraints