labor-arbitration-system/nlp-service/training/train_with_split.py

#!/usr/bin/env python3
"""Train BERT multi-task model with explicit 90/10 train/test split on real case data."""

from __future__ import annotations

import json
import os
import random
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

# NOTE: transformers must import before torch on Windows
from transformers import AutoTokenizer, get_linear_schedule_with_warmup
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm

from app.services.bert_multi_task_model import ChineseRobertaMultiTask
from train_bert import LaborCaseDataset, collate_batch, train_epoch, evaluate

DATASET = Path(__file__).resolve().parent.parent.parent / "backend" / "data" / "training_dataset.json"
OUTPUT = Path(__file__).resolve().parent.parent / "models" / "chinese_roberta_labor_extractor_v2"
MODEL_NAME = "hfl/chinese-roberta-wwm-ext"
EPOCHS = 10
BATCH_SIZE = 4
LR = 2e-5
MAX_LENGTH = 512
PATIENCE = 3
TEST_RATIO = 0.1  # 10% for test

random.seed(42)
torch.manual_seed(42)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Device: {device}")

# Load data
with open(DATASET, encoding="utf-8") as f:
    data = json.load(f)

cases = data["dataset"]
random.shuffle(cases)

n_test = max(1, int(len(cases) * TEST_RATIO))
n_train = len(cases) - n_test

train_cases = cases[:n_train]
test_cases = cases[n_train:]

print(f"Total: {len(cases)}, Train: {n_train}, Test: {n_test}")

# Tokenizer
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)

# Datasets
train_ds = LaborCaseDataset(train_cases, tokenizer, MAX_LENGTH, for_training=True)
test_ds = LaborCaseDataset(test_cases, tokenizer, MAX_LENGTH, for_training=True)

train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True,
                          collate_fn=collate_batch, num_workers=0)
test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False,
                         collate_fn=collate_batch, num_workers=0)

# Model
model = ChineseRobertaMultiTask(model_name=MODEL_NAME)
model.to(device)
print(f"Model params: {sum(p.numel() for p in model.parameters()):,}")

# Optimizer
no_decay = ["bias", "LayerNorm.weight"]
opt_params = [
    {"params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
     "weight_decay": 0.01},
    {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
     "weight_decay": 0.0},
]
optimizer = torch.optim.AdamW(opt_params, lr=LR)
total_steps = len(train_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=int(total_steps * 0.1),
                                            num_training_steps=total_steps)

# Training
best_loss = float("inf")
patience_cnt = 0

for epoch in range(1, EPOCHS + 1):
    train_loss = train_epoch(model, train_loader, optimizer, scheduler, device, epoch)
    dev_metrics = evaluate(model, test_loader, device)

    print(f"Epoch {epoch:>2}: train_loss={train_loss:.4f}, "
          f"test_loss={dev_metrics['loss']:.4f}, "
          f"test_cause_acc={dev_metrics['cause_accuracy']:.3f}")

    if dev_metrics["loss"] < best_loss:
        best_loss = dev_metrics["loss"]
        patience_cnt = 0
        os.makedirs(OUTPUT, exist_ok=True)
        model.save_pretrained(str(OUTPUT))
        tokenizer.save_pretrained(str(OUTPUT))
        print(f"  -> Saved to {OUTPUT}")
    else:
        patience_cnt += 1
        if patience_cnt >= PATIENCE:
            print(f"Early stop at epoch {epoch}")
            break

print(f"\nDone. Best test loss: {best_loss:.4f}")
print(f"Model: {OUTPUT}")