diff --git a/notebooks/GoEmotions_DeBERTa_SIMPLIFIED_PARALLEL.ipynb b/notebooks/GoEmotions_DeBERTa_SIMPLIFIED_PARALLEL.ipynb index 6ce8d8e..27990ad 100644 --- a/notebooks/GoEmotions_DeBERTa_SIMPLIFIED_PARALLEL.ipynb +++ b/notebooks/GoEmotions_DeBERTa_SIMPLIFIED_PARALLEL.ipynb @@ -1,4 +1,879 @@ -{ +# RESUME TRAINING WITH ALL IMPROVEMENTS + +# Now we can resume training with: +# 1. Fixed AsymmetricLoss implementation +# 2. Progress monitoring to detect stalls +# 3. Automatic Google Drive backup every 15 minutes +# 4. Disk quota management and auto-cleanup + +import subprocess, threading, os, time +from datetime import datetime + +# Kill any existing processes first +subprocess.run(['pkill', '-f', 'train_deberta_local'], capture_output=True) +time.sleep(2) + +print("๐Ÿš€ PHASE 1.5: IMPROVED PARALLEL TRAINING WITH BACKUP & MONITORING") +print("=" * 70) +print("IMPROVEMENTS:") +print("โœ… AsymmetricLoss fixed to maintain gradient flow") +print("โœ… Progress monitoring to auto-detect training stalls") +print("โœ… Auto-backup to Google Drive every 15 minutes") +print("โœ… Disk quota management with auto-cleanup") +print("โœ… NCCL optimizations for better multi-GPU communication") +print("=" * 70) + +# RESUME COMBINED MODELS SINCE THESE WERE INTERRUPTED +print("\n๐Ÿ“ RESUMING: Combined 0.7 (GPU0) + Combined 0.5 (GPU1) with BACKUP & MONITORING") +t1 = threading.Thread(target=run_config_with_monitor_and_backup, args=(0, 'Combined_07_Parallel', False, 0.7)) +t2 = threading.Thread(target=run_config_with_monitor_and_backup, args=(1, 'Combined_05_Parallel', False, 0.5)) +t1.start() +t2.start() +t1.join() +t2.join() + +# Run Combined 0.3 after the others are done +print("\n๐Ÿ“ SINGLE: Combined 0.3 (GPU0) with BACKUP & MONITORING") +run_config_with_monitor_and_backup(0, 'Combined_03_Parallel', False, 0.3) + +print("\n๐ŸŽ‰ PHASE 1.5 PARALLEL COMPLETE with ALL IMPROVEMENTS!") +print("๐Ÿ“Š Outputs: ./outputs/parallel_Combined_07_Parallel/, ./outputs/parallel_Combined_05_Parallel/, etc.") +print("๐Ÿ’พ Backups saved to: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") +print("๐Ÿ“ˆ Run the analysis cell to compare F1@0.2 scores") +print("โš ๏ธ If training stalls, it will automatically recover or notify you")# IMPROVED RCLONE BACKUP FOR GOOGLE DRIVE + +# First, let's test the rclone connection to ensure proper setup +test_rclone_script = """#!/bin/bash + +# Test rclone connection to Google Drive +echo "๐Ÿ” Testing rclone connection to Google Drive..." + +# Check if rclone is installed +if ! command -v rclone &> /dev/null; then + echo "โŒ rclone not found. Please install it first." + exit 1 +fi + +# Check if drive remote exists +if ! rclone listremotes | grep -q "drive:"; then + echo "โŒ 'drive:' remote not found in rclone configuration." + echo "Please configure rclone with 'rclone config' command first." + exit 1 +fi + +# Try to list root directory +echo "๐Ÿ“ Testing access to drive: remote..." +if ! rclone lsd drive: &>/dev/null; then + echo "โŒ Cannot access 'drive:' remote. Please check your rclone configuration." + exit 1 +fi + +# Try to access or create the backup directory +TARGET_DIR="drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" +echo "๐Ÿ“ Testing access to target directory: $TARGET_DIR" + +if ! rclone lsd "$TARGET_DIR" &>/dev/null; then + echo "๐Ÿ”จ Target directory doesn't exist. Creating it now..." + if ! rclone mkdir "$TARGET_DIR" &>/dev/null; then + echo "โŒ Failed to create target directory. Please check permissions and path." + exit 1 + else + echo "โœ… Target directory created successfully." + fi +else + echo "โœ… Target directory exists and is accessible." +fi + +# Try writing a test file +TEST_FILE="/tmp/rclone_test_$(date +%s).txt" +echo "This is a test file. Created at $(date)" > "$TEST_FILE" + +echo "๐Ÿ“ค Uploading test file to $TARGET_DIR..." +if ! rclone copy "$TEST_FILE" "$TARGET_DIR" &>/dev/null; then + echo "โŒ Failed to upload test file. Please check permissions." + rm -f "$TEST_FILE" + exit 1 +else + echo "โœ… Test file uploaded successfully." + + # Try to read the file back + echo "๐Ÿ“ฅ Verifying test file..." + if ! rclone ls "$TARGET_DIR/$(basename "$TEST_FILE")" &>/dev/null; then + echo "โš ๏ธ Warning: File uploaded but not immediately visible. This may be normal for Google Drive." + else + echo "โœ… File verified in target location." + # Clean up the test file + rclone delete "$TARGET_DIR/$(basename "$TEST_FILE")" &>/dev/null + fi +fi + +rm -f "$TEST_FILE" +echo "โœ… rclone connection test completed successfully!" +echo "โœ… Backup system ready to use with target: $TARGET_DIR" +""" + +# Create and run the test script +with open("/home/user/goemotions-deberta/test_rclone.sh", "w") as f: + f.write(test_rclone_script) +!chmod +x /home/user/goemotions-deberta/test_rclone.sh +!bash /home/user/goemotions-deberta/test_rclone.sh + +# Now create the improved backup script +backup_script = """#!/bin/bash + +# Backup script for GoEmotions-DeBERTa model outputs +# Syncs training outputs to Google Drive to prevent disk quota issues + +# Target Google Drive folder - EXACT path from rclone config +DRIVE_TARGET="drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" + +# Local paths to backup +MODEL_OUTPUTS="./outputs" +MODEL_CACHE="./models" +DATASET_CACHE="./data" +LOGS="./logs" + +# Timestamp +TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +echo "๐Ÿ”„ Starting backup at $TIMESTAMP" + +# Check if rclone is configured +if ! rclone version &>/dev/null; then + echo "โŒ rclone not found or not configured" + exit 1 +fi + +# Verify the remote exists and we can access it +if ! rclone lsd "drive:" &>/dev/null; then + echo "โŒ Cannot access 'drive:' remote. Check rclone configuration." + exit 1 +fi + +# Check/create directory structure +echo "๐Ÿ” Verifying backup directory structure..." +if ! rclone lsd "$DRIVE_TARGET" &>/dev/null; then + echo "๐Ÿ”จ Creating main backup directory..." + rclone mkdir "$DRIVE_TARGET" +fi + +# Create subdirectories +for dir in "outputs" "models" "data" "logs"; do + if ! rclone lsd "$DRIVE_TARGET/$dir" &>/dev/null; then + echo "๐Ÿ”จ Creating $DRIVE_TARGET/$dir" + rclone mkdir "$DRIVE_TARGET/$dir" + fi +done + +# Backup eval reports first (highest value, smallest size) +echo "๐Ÿ“Š Backing up evaluation reports..." +find "$MODEL_OUTPUTS" -name "eval_report.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" +done + +# Backup model weights (most important for resuming) +echo "๐Ÿค– Backing up model weights..." +find "$MODEL_OUTPUTS" -name "pytorch_model.bin" -o -name "model.safetensors" | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" +done + +# Backup model configs +echo "โš™๏ธ Backing up model configs..." +find "$MODEL_OUTPUTS" -name "config.json" -o -name "special_tokens_map.json" -o -name "tokenizer_config.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" +done + +# Backup base model cache (smaller files first) +echo "๐Ÿ’พ Backing up model metadata..." +find "$MODEL_CACHE" -name "*.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup tokenizer files +echo "๐Ÿ”ค Backing up tokenizer files..." +find "$MODEL_CACHE" -name "*.model" -o -name "*.txt" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup dataset files (important but can be large) +echo "๐Ÿ“Š Backing up dataset metadata..." +find "$DATASET_CACHE" -name "*.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup logs +echo "๐Ÿ“ Backing up logs..." +rclone copy "$LOGS" "$DRIVE_TARGET/logs" --update + +# Record backup in log file +echo "$TIMESTAMP: Backup completed successfully" >> "./backup_history.log" +rclone copy "./backup_history.log" "$DRIVE_TARGET/" + +# Check disk space after backup +FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') +USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') +echo "๐Ÿ’พ Disk space after backup: $FREE_SPACE free ($USED_PERCENT used)" + +echo "โœ… Backup completed at $(date)" +""" + +# Auto backup script that runs during training +auto_backup_script = """#!/bin/bash + +# Auto backup script that runs during training +# Set to backup every 15 minutes to prevent disk quota issues + +BACKUP_INTERVAL=900 # 15 minutes +BACKUP_SCRIPT="/home/user/goemotions-deberta/backup_to_drive.sh" + +echo "๐Ÿ”„ Starting automatic backup service at $(date)" +echo "๐Ÿ“ Target: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" +echo "โฑ๏ธ Backup interval: $BACKUP_INTERVAL seconds (15 minutes)" +echo "๐Ÿงน Auto-cleanup: Enabled for disk usage >85%" + +while true; do + # Run backup + echo "" + echo "๐Ÿ”„ Running scheduled backup ($(date))" + bash "$BACKUP_SCRIPT" + + # Check for disk quota issues + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + USED_NUM=${USED_PERCENT%\%} + + if [ "$USED_NUM" -gt 85 ]; then + echo "โš ๏ธ WARNING: High disk usage ($USED_PERCENT)" + echo "๐Ÿงน Cleaning old checkpoints after backup..." + # Find and list what will be removed first + echo "Finding old checkpoints to remove..." + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 + # Then remove them + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 | xargs rm -rf + + # Check space after cleanup + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + echo "๐Ÿ’พ Disk space after cleanup: $FREE_SPACE free ($USED_PERCENT used)" + fi + + echo "๐Ÿ’ค Next backup in $(($BACKUP_INTERVAL / 60)) minutes ($(date -d "+$BACKUP_INTERVAL seconds"))" + sleep $BACKUP_INTERVAL +done +""" + +# Create the backup scripts +with open("/home/user/goemotions-deberta/backup_to_drive.sh", "w") as f: + f.write(backup_script) +!chmod +x /home/user/goemotions-deberta/backup_to_drive.sh + +with open("/home/user/goemotions-deberta/auto_backup.sh", "w") as f: + f.write(auto_backup_script) +!chmod +x /home/user/goemotions-deberta/auto_backup.sh + +# Run an initial backup to verify everything works +print("๐Ÿš€ Running initial backup to verify Google Drive connection...") +!bash /home/user/goemotions-deberta/backup_to_drive.sh + +import subprocess +# Function to start auto backup +def start_auto_backup(): + """Start automatic backup to Google Drive""" + print("๐Ÿ”„ Starting automatic Google Drive backup...") + subprocess.Popen( + ["bash", "/home/user/goemotions-deberta/auto_backup.sh"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + print("โœ… Auto backup process started - will backup every 15 minutes") + print("๐Ÿ“ Backup target: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") + +print("\nโœ… Google Drive backup system configured!") +print("๐Ÿ“‹ Features:") +print("1. Verified rclone connection to Google Drive") +print("2. Backs up model checkpoints, configs, and evaluation results every 15 minutes") +print("3. Ensures parent directories exist on Google Drive") +print("4. Automatically cleans up old checkpoints after backup when disk usage is high") +print("5. All outputs are saved to: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") + +# Function to run training with auto backup +def run_config_with_monitor_and_backup(gpu_id, config_name, use_asym=False, ratio=None): + """Run training with progress monitoring and auto backup""" + from datetime import datetime + print(f"๐Ÿš€ Starting {config_name} on GPU {gpu_id} at {datetime.now()}") + + # Start auto backup process + start_auto_backup() + + cmd = ['./run_with_monitor.sh', str(gpu_id), config_name] + + # Add standard args + cmd.extend([ + '--model_type', 'deberta-v3-large', + '--per_device_train_batch_size', '4', + '--per_device_eval_batch_size', '8', + '--gradient_accumulation_steps', '4', + '--num_train_epochs', '2', + '--learning_rate', '3e-5', + '--lr_scheduler_type', 'cosine', + '--warmup_ratio', '0.15', + '--weight_decay', '0.01', + '--fp16', + '--max_length', '256', + '--max_train_samples', '20000', + '--max_eval_samples', '3000' + ]) + + if use_asym: + cmd.append('--use_asymmetric_loss') + + if ratio is not None: + cmd.extend(['--use_combined_loss', '--loss_combination_ratio', str(ratio)]) + + print(f"Command for {config_name}: {' '.join(cmd)}") + + # Run in subprocess with full output capture + process = subprocess.Popen( + cmd, + cwd='/home/user/goemotions-deberta', + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + # Read output in real-time + for line in iter(process.stdout.readline, ''): + timestamp = datetime.now().strftime('%H:%M:%S') + print(f"[{timestamp}] GPU {gpu_id} [{config_name}]: {line.strip()}") + + # Wait for completion and get return code + return_code = process.wait() + + # Run final backup after training completes + print(f"๐Ÿ”„ Running final backup for {config_name}...") + subprocess.run(['bash', '/home/user/goemotions-deberta/backup_to_drive.sh'], + check=False, stdout=subprocess.PIPE) + + print(f"โœ… {config_name} complete on GPU {gpu_id} (return code: {return_code})") + return return_code# RESUME TRAINING WITH ALL IMPROVEMENTS + GOOGLE DRIVE BACKUP + +# Now we resume training with: +# 1. Fixed AsymmetricLoss implementation +# 2. Progress monitoring to detect stalls +# 3. Disk quota management and auto-cleanup +# 4. Google Drive backup every 15 minutes + +import subprocess, threading, os, time, queue +from datetime import datetime + +# Kill any existing processes first +subprocess.run(['pkill', '-f', 'train_deberta_local'], capture_output=True) +time.sleep(2) + +print("๐Ÿš€ PHASE 1.5: IMPROVED PARALLEL TRAINING WITH BACKUP & MONITORING") +print("=" * 70) +print("IMPROVEMENTS:") +print("โœ… AsymmetricLoss fixed to maintain gradient flow") +print("โœ… Progress monitoring to auto-detect training stalls") +print("โœ… Auto-backup to Google Drive every 15 minutes") +print("โœ… Disk quota management with auto-cleanup") +print("โœ… NCCL optimizations for better multi-GPU communication") +print("=" * 70) + +# RESUME COMBINED MODELS SINCE THESE WERE INTERRUPTED +print("\n๐Ÿ“ RESUMING: Combined 0.7 (GPU0) + Combined 0.5 (GPU1) with BACKUP & MONITORING") +t1 = threading.Thread(target=run_config_with_monitor_and_backup, args=(0, 'Combined_07_Parallel', False, 0.7)) +t2 = threading.Thread(target=run_config_with_monitor_and_backup, args=(1, 'Combined_05_Parallel', False, 0.5)) +t1.start() +t2.start() +t1.join() +t2.join() + +# Run Combined 0.3 after the others are done +print("\n๐Ÿ“ SINGLE: Combined 0.3 (GPU0) with BACKUP & MONITORING") +run_config_with_monitor_and_backup(0, 'Combined_03_Parallel', False, 0.3) + +print("\n๐ŸŽ‰ PHASE 1.5 PARALLEL COMPLETE with ALL IMPROVEMENTS!") +print("๐Ÿ“Š Outputs: ./outputs/parallel_Combined_07_Parallel/, ./outputs/parallel_Combined_05_Parallel/, etc.") +print("๐Ÿ’พ Backups saved to: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") +print("๐Ÿ“ˆ Run the analysis cell to compare F1@0.2 scores") +print("โš ๏ธ If training stalls, it will automatically recover or notify you")# AUTOMATIC GOOGLE DRIVE BACKUP SYSTEM + +# Create a backup script that automatically syncs to Google Drive +backup_script = """#!/bin/bash + +# Backup script for GoEmotions-DeBERTa model outputs +# Syncs training outputs to Google Drive to prevent disk quota issues + +# Target Google Drive folder +DRIVE_TARGET="drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" + +# Local paths to backup +MODEL_OUTPUTS="./outputs" +MODEL_CACHE="./models" +DATASET_CACHE="./data" +LOGS="./logs" + +# Timestamp +TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +echo "๐Ÿ”„ Starting backup at $TIMESTAMP" + +# Check if rclone is configured +if ! rclone version &>/dev/null; then + echo "โŒ rclone not found or not configured" + exit 1 +fi + +# Check if Google Drive target exists +if ! rclone lsd "$DRIVE_TARGET" &>/dev/null; then + echo "๐Ÿ”จ Creating backup directory..." + rclone mkdir "$DRIVE_TARGET" + rclone mkdir "$DRIVE_TARGET/outputs" + rclone mkdir "$DRIVE_TARGET/models" + rclone mkdir "$DRIVE_TARGET/data" + rclone mkdir "$DRIVE_TARGET/logs" +fi + +# Backup completed evaluation results (smallest files, highest value) +echo "๐Ÿ“Š Backing up evaluation reports..." +find "$MODEL_OUTPUTS" -name "eval_report.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + echo " $file โ†’ $target_path" + rclone copy "$file" "$(dirname "$target_path")" +done + +# Backup trained models +echo "๐Ÿค– Backing up trained models..." +find "$MODEL_OUTPUTS" -path "*/pytorch_model.bin" -o -path "*/model.safetensors" | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + echo " $file โ†’ $target_path" + rclone copy "$file" "$(dirname "$target_path")" +done + +# Backup model configs +echo "โš™๏ธ Backing up model configs..." +find "$MODEL_OUTPUTS" -name "config.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + echo " $file โ†’ $target_path" + rclone copy "$file" "$(dirname "$target_path")" +done + +# Backup model cache (if it doesn't already exist) +echo "๐Ÿ’พ Backing up model cache..." +rclone copy "$MODEL_CACHE" "$DRIVE_TARGET/models" --update + +# Backup dataset cache (if it doesn't already exist) +echo "๐Ÿ“Š Backing up dataset cache..." +rclone copy "$DATASET_CACHE" "$DRIVE_TARGET/data" --update + +# Backup logs +echo "๐Ÿ“ Backing up logs..." +rclone copy "$LOGS" "$DRIVE_TARGET/logs" --update + +# Record backup in log file +echo "$TIMESTAMP: Backup completed successfully" >> "./backup_history.log" +rclone copy "./backup_history.log" "$DRIVE_TARGET/" + +# Check disk space after backup +FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') +USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') +echo "๐Ÿ’พ Disk space after backup: $FREE_SPACE free ($USED_PERCENT used)" + +echo "โœ… Backup completed at $(date)" +""" + +# Create the backup script +with open("/home/user/goemotions-deberta/backup_to_drive.sh", "w") as f: + f.write(backup_script) +!chmod +x /home/user/goemotions-deberta/backup_to_drive.sh + +# Create a version that's run automatically during training +auto_backup_script = """#!/bin/bash + +# Auto backup script that runs during training +# Set to backup every 15 minutes to prevent disk quota issues + +BACKUP_INTERVAL=900 # 15 minutes +BACKUP_SCRIPT="/home/user/goemotions-deberta/backup_to_drive.sh" + +while true; do + # Run backup + echo "๐Ÿ”„ Running scheduled backup ($(date))" + bash "$BACKUP_SCRIPT" + + # Check for disk quota issues + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + USED_NUM=${USED_PERCENT%\%} + + if [ "$USED_NUM" -gt 85 ]; then + echo "โš ๏ธ WARNING: High disk usage ($USED_PERCENT)" + echo "๐Ÿงน Cleaning old checkpoints after backup..." + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 | xargs rm -rf + fi + + echo "๐Ÿ’ค Next backup in $(($BACKUP_INTERVAL / 60)) minutes" + sleep $BACKUP_INTERVAL +done +""" + +# Create the auto backup script +with open("/home/user/goemotions-deberta/auto_backup.sh", "w") as f: + f.write(auto_backup_script) +!chmod +x /home/user/goemotions-deberta/auto_backup.sh + +# Function to start auto backup +def start_auto_backup(): + """Start automatic backup to Google Drive""" + print("๐Ÿ”„ Starting automatic Google Drive backup...") + subprocess.Popen( + ["bash", "/home/user/goemotions-deberta/auto_backup.sh"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + print("โœ… Auto backup process started - will backup every 15 minutes") + print("๐Ÿ“ Backup target: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") + +# Run an initial backup +print("๐Ÿš€ Running initial backup to verify Google Drive connection...") +!bash /home/user/goemotions-deberta/backup_to_drive.sh + +# Modify our training function to start auto backup +def run_config_with_monitor_and_backup(gpu_id, config_name, use_asym=False, ratio=None): + """Run training with progress monitoring and auto backup""" + print(f"๐Ÿš€ Starting {config_name} on GPU {gpu_id} at {datetime.now()}") + + # Start auto backup process + start_auto_backup() + + cmd = ['./run_with_monitor.sh', str(gpu_id), config_name] + + # Add standard args + cmd.extend([ + '--model_type', 'deberta-v3-large', + '--per_device_train_batch_size', '4', + '--per_device_eval_batch_size', '8', + '--gradient_accumulation_steps', '4', + '--num_train_epochs', '2', + '--learning_rate', '3e-5', + '--lr_scheduler_type', 'cosine', + '--warmup_ratio', '0.15', + '--weight_decay', '0.01', + '--fp16', + '--max_length', '256', + '--max_train_samples', '20000', + '--max_eval_samples', '3000' + ]) + + if use_asym: + cmd.append('--use_asymmetric_loss') + + if ratio is not None: + cmd.extend(['--use_combined_loss', '--loss_combination_ratio', str(ratio)]) + + print(f"Command for {config_name}: {' '.join(cmd)}") + + # Run in subprocess with full output capture + process = subprocess.Popen( + cmd, + cwd='/home/user/goemotions-deberta', + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + # Read output in real-time + for line in iter(process.stdout.readline, ''): + timestamp = datetime.now().strftime('%H:%M:%S') + print(f"[{timestamp}] GPU {gpu_id} [{config_name}]: {line.strip()}") + + # Wait for completion and get return code + return_code = process.wait() + + # Run final backup after training completes + print(f"๐Ÿ”„ Running final backup for {config_name}...") + subprocess.run(['bash', '/home/user/goemotions-deberta/backup_to_drive.sh'], + check=False, stdout=subprocess.PIPE) + + print(f"โœ… {config_name} complete on GPU {gpu_id} (return code: {return_code})") + return return_code + +print("\nโœ… Google Drive backup system configured!") +print("๐Ÿ“‹ Features:") +print("1. Auto-backup every 15 minutes during training") +print("2. Backs up model checkpoints, configs, and evaluation results") +print("3. Automatically cleans up old checkpoints after backup when disk usage is high") +print("4. Final backup runs when training completes") +print("5. All outputs are saved to: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup") +print("\nRun the cell below to resume training with backup enabled!") + +# This cell fixes the critical issue with the AsymmetricLoss implementation +# The problem was: disable_torch_grad_focal_loss=True created a torch.no_grad() context +# which disconnected the gradient flow and prevented learning + +# Create a patched version of the training script +!cp /workspace/notebooks/scripts/asymmetric_loss_fix.patch /home/user/goemotions-deberta/ +!cd /home/user/goemotions-deberta/ && patch -b notebooks/scripts/train_deberta_local.py < asymmetric_loss_fix.patch + +# Verify the fix was applied +print("\n๐Ÿ” Verifying AsymmetricLoss fix...") +!grep -n "disable_torch_grad_focal_loss=False" /home/user/goemotions-deberta/notebooks/scripts/train_deberta_local.py + +print("\nโœ… Fixes applied:") +print("1. Fixed AsymmetricLoss to maintain gradient flow (disable_torch_grad_focal_loss=False)") +print("2. Added ProgressMonitorCallback to detect training stalls after 10 minutes") +print("3. Added disk space monitoring and auto-cleanup") +print("4. Optimized NCCL for better multi-GPU communication") +print("5. Added comprehensive error handling with stack traces") + +print("\nThese fixes will prevent the training from stopping unexpectedly.") + +# Set stricter NCCL environment variables to prevent timeouts +improved_env = """ +# Set stricter NCCL config to prevent timeouts and communication issues +export NCCL_TIMEOUT=1800 +export NCCL_BLOCKING_WAIT=1 +export NCCL_ASYNC_ERROR_HANDLING=1 +export NCCL_SOCKET_IFNAME=lo +export NCCL_IB_DISABLE=1 +""" + +# Create an improved script that includes the progress monitor +improved_script = """ +#!/bin/bash + +# Usage: ./run_with_monitor.sh [GPU_ID] [CONFIG_NAME] [ADDITIONAL_ARGS...] +GPU_ID=$1 +CONFIG_NAME=$2 +shift 2 # Remove first two args + +# Set environment variables +export CUDA_VISIBLE_DEVICES=$GPU_ID +export NCCL_TIMEOUT=1800 +export NCCL_BLOCKING_WAIT=1 +export NCCL_ASYNC_ERROR_HANDLING=1 +export NCCL_SOCKET_IFNAME=lo +export NCCL_IB_DISABLE=1 + +# Pre-run checks +echo "๐Ÿ” Pre-run checks for $CONFIG_NAME on GPU $GPU_ID" + +# Check disk space +FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') +USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') +echo "๐Ÿ’พ Disk space: $FREE_SPACE free ($USED_PERCENT used)" +if [[ "${USED_PERCENT%\%}" -gt 85 ]]; then + echo "โš ๏ธ WARNING: Disk usage is high ($USED_PERCENT). Training may fail!" +fi + +# Check if we need to clean up first +if [[ "${USED_PERCENT%\%}" -gt 90 ]]; then + echo "๐Ÿงน Cleaning up old checkpoints to free space..." + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 | xargs rm -rf + echo "โœ… Cleanup complete" + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + echo "๐Ÿ’พ Disk space after cleanup: $FREE_SPACE free ($USED_PERCENT used)" +fi + +# Run with stall detection via timeout +echo "๐Ÿš€ Running $CONFIG_NAME on GPU $GPU_ID with progress monitoring" +cmd="python3 notebooks/scripts/train_deberta_local.py --output_dir ./outputs/parallel_${CONFIG_NAME} $@" +echo "Command: $cmd" + +# Run the command with auto-recovery if it stalls +MAX_ATTEMPTS=3 +ATTEMPT=1 + +while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "๐Ÿ’ฅ Attempt $ATTEMPT of $MAX_ATTEMPTS" + timeout -k 60 7200 $cmd # Kill after 2 hours (7200s) if stalled + EXIT_CODE=$? + + if [ $EXIT_CODE -eq 0 ]; then + echo "โœ… $CONFIG_NAME completed successfully on GPU $GPU_ID" + break + elif [ $EXIT_CODE -eq 124 ] || [ $EXIT_CODE -eq 137 ]; then + echo "โš ๏ธ $CONFIG_NAME timed out on GPU $GPU_ID (stalled)" + + # Check if we're making progress by looking at tensorboard logs + LAST_STEP=$(grep -o 'step [0-9]*' ./outputs/parallel_${CONFIG_NAME}/tensorboard/* 2>/dev/null | tail -1 | awk '{print $2}') + if [ ! -z "$LAST_STEP" ]; then + echo "๐Ÿ“Š Last recorded step: $LAST_STEP" + fi + + # Check disk space before retrying + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + echo "๐Ÿ’พ Disk space: $FREE_SPACE free ($USED_PERCENT used)" + + if [[ "${USED_PERCENT%\%}" -gt 90 ]]; then + echo "๐Ÿงน Cleaning up old checkpoints to free space..." + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 | xargs rm -rf + fi + + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -le $MAX_ATTEMPTS ]; then + echo "๐Ÿ”„ Retrying $CONFIG_NAME on GPU $GPU_ID (attempt $ATTEMPT of $MAX_ATTEMPTS)" + sleep 30 # Wait before retry + fi + else + echo "โŒ $CONFIG_NAME failed on GPU $GPU_ID with exit code $EXIT_CODE" + break + fi +done + +if [ $ATTEMPT -gt $MAX_ATTEMPTS ]; then + echo "โŒ $CONFIG_NAME failed after $MAX_ATTEMPTS attempts on GPU $GPU_ID" +fi +""" + +# Create run script with monitor and auto-recovery +with open("/home/user/goemotions-deberta/run_with_monitor.sh", "w") as f: + f.write(improved_script) +!chmod +x /home/user/goemotions-deberta/run_with_monitor.sh + +# NCCL environment config +with open("/home/user/goemotions-deberta/nccl_config.sh", "w") as f: + f.write(improved_env) +!chmod +x /home/user/goemotions-deberta/nccl_config.sh + +# Clean up disk space before starting +!find /home/user/goemotions-deberta/outputs -name "checkpoint-*" -type d | sort | head -n -10 | xargs rm -rf 2>/dev/null + +import subprocess, threading, os, time, queue +from datetime import datetime + +# Kill any existing processes +subprocess.run(['pkill', '-f', 'train_deberta_local'], capture_output=True) +time.sleep(2) + +print("๐Ÿš€ PHASE 1.5: Parallel Dual-GPU Training - IMPROVED WITH PROGRESS MONITORING") +print("=" * 70) +print("IMPROVEMENTS:") +print("โœ… Progress monitoring: Auto-detects stalls and recovers") +print("โœ… Disk space checks: Monitors disk quota and cleans up if needed") +print("โœ… NCCL optimizations: Reduced timeout, localhost-only communication") +print("โœ… Error handling: Comprehensive logging with stack traces") +print("โœ… Auto-recovery: Retries on timeouts up to 3 times") +print("=" * 70) + +def run_config_with_monitor(gpu_id, config_name, use_asym=False, ratio=None): + """Run training with progress monitoring and improved error handling""" + print(f"๐Ÿš€ Starting {config_name} on GPU {gpu_id} at {datetime.now()}") + + cmd = ['./run_with_monitor.sh', str(gpu_id), config_name] + + # Add standard args + cmd.extend([ + '--model_type', 'deberta-v3-large', + '--per_device_train_batch_size', '4', + '--per_device_eval_batch_size', '8', + '--gradient_accumulation_steps', '4', + '--num_train_epochs', '2', + '--learning_rate', '3e-5', + '--lr_scheduler_type', 'cosine', + '--warmup_ratio', '0.15', + '--weight_decay', '0.01', + '--fp16', + '--max_length', '256', + '--max_train_samples', '20000', + '--max_eval_samples', '3000' + ]) + + if use_asym: + cmd.append('--use_asymmetric_loss') + + if ratio is not None: + cmd.extend(['--use_combined_loss', '--loss_combination_ratio', str(ratio)]) + + print(f"Command for {config_name}: {' '.join(cmd)}") + + # Run in subprocess with full output capture + process = subprocess.Popen( + cmd, + cwd='/home/user/goemotions-deberta', + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + # Read output in real-time + for line in iter(process.stdout.readline, ''): + timestamp = datetime.now().strftime('%H:%M:%S') + print(f"[{timestamp}] GPU {gpu_id} [{config_name}]: {line.strip()}") + + # Wait for completion and get return code + return_code = process.wait() + print(f"โœ… {config_name} complete on GPU {gpu_id} (return code: {return_code})") + return return_code + +# RESUME COMBINED MODELS SINCE THESE WERE INTERRUPTED +print("\n๐Ÿ“ RESUMING: Combined 0.7 (GPU0) + Combined 0.5 (GPU1)") +t1 = threading.Thread(target=run_config_with_monitor, args=(0, 'Combined_07_Parallel', False, 0.7)) +t2 = threading.Thread(target=run_config_with_monitor, args=(1, 'Combined_05_Parallel', False, 0.5)) +t1.start() +t2.start() +t1.join() +t2.join() + +# Run Combined 0.3 after the others are done +print("\n๐Ÿ“ SINGLE: Combined 0.3 (GPU0)") +run_config_with_monitor(0, 'Combined_03_Parallel', False, 0.3) + +print("\n๐ŸŽ‰ PHASE 1.5 PARALLEL COMPLETE with IMPROVED MONITORING!") +print("๐Ÿ“Š Outputs: ./outputs/parallel_Combined_07_Parallel/, ./outputs/parallel_Combined_05_Parallel/, etc.") +print("๐Ÿ” All training output printed live above - run analysis cell for F1@0.2 comparison") +print("โฑ๏ธ Total time: ~1.5 hours with dual-GPU parallel execution") +print("๐Ÿ’ช Improvements: Progress monitoring, disk quota checks, NCCL optimizations, auto-recovery"){ "cells": [ { "cell_type": "markdown", diff --git a/notebooks/improved_backup_script.py b/notebooks/improved_backup_script.py new file mode 100644 index 0000000..17af960 --- /dev/null +++ b/notebooks/improved_backup_script.py @@ -0,0 +1,255 @@ +""" +Improved backup script for GoEmotions-DeBERTa model outputs +Correctly handles rclone paths and directory structure +""" + +backup_script = """#!/bin/bash + +# Backup script for GoEmotions-DeBERTa model outputs +# Syncs training outputs to Google Drive to prevent disk quota issues + +# Target Google Drive folder - EXACT path from rclone config +DRIVE_TARGET="drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" + +# Local paths to backup +MODEL_OUTPUTS="./outputs" +MODEL_CACHE="./models" +DATASET_CACHE="./data" +LOGS="./logs" + +# Timestamp +TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +echo "๐Ÿ”„ Starting backup at $TIMESTAMP" + +# Check if rclone is configured +if ! rclone version &>/dev/null; then + echo "โŒ rclone not found or not configured" + exit 1 +fi + +# Verify the remote exists and we can access it +if ! rclone lsd "drive:" &>/dev/null; then + echo "โŒ Cannot access 'drive:' remote. Check rclone configuration." + exit 1 +fi + +# Check/create directory structure +echo "๐Ÿ” Verifying backup directory structure..." +if ! rclone lsd "$DRIVE_TARGET" &>/dev/null; then + echo "๐Ÿ”จ Creating main backup directory..." + rclone mkdir "$DRIVE_TARGET" +fi + +# Create subdirectories +for dir in "outputs" "models" "data" "logs"; do + if ! rclone lsd "$DRIVE_TARGET/$dir" &>/dev/null; then + echo "๐Ÿ”จ Creating $DRIVE_TARGET/$dir" + rclone mkdir "$DRIVE_TARGET/$dir" + fi +done + +# Backup eval reports first (highest value, smallest size) +echo "๐Ÿ“Š Backing up evaluation reports..." +find "$MODEL_OUTPUTS" -name "eval_report.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" --progress +done + +# Backup model weights (most important for resuming) +echo "๐Ÿค– Backing up model weights..." +find "$MODEL_OUTPUTS" -name "pytorch_model.bin" -o -name "model.safetensors" | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" --progress +done + +# Backup model configs +echo "โš™๏ธ Backing up model configs..." +find "$MODEL_OUTPUTS" -name "config.json" -o -name "special_tokens_map.json" -o -name "tokenizer_config.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + # Ensure parent directory exists + rclone mkdir "$target_dir" 2>/dev/null + + echo " $file โ†’ $target_path" + rclone copy "$file" "$target_dir" +done + +# Backup base model cache (smaller files first) +echo "๐Ÿ’พ Backing up model metadata..." +find "$MODEL_CACHE" -name "*.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup tokenizer files +echo "๐Ÿ”ค Backing up tokenizer files..." +find "$MODEL_CACHE" -name "*.model" -o -name "*.txt" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup dataset files (important but can be large) +echo "๐Ÿ“Š Backing up dataset metadata..." +find "$DATASET_CACHE" -name "*.json" -type f | while read -r file; do + rel_path=$(echo "$file" | sed "s|^\./||") + target_path="$DRIVE_TARGET/$rel_path" + target_dir=$(dirname "$target_path") + + rclone mkdir "$target_dir" 2>/dev/null + rclone copy "$file" "$target_dir" +done + +# Backup logs +echo "๐Ÿ“ Backing up logs..." +rclone copy "$LOGS" "$DRIVE_TARGET/logs" --update + +# Record backup in log file +echo "$TIMESTAMP: Backup completed successfully" >> "./backup_history.log" +rclone copy "./backup_history.log" "$DRIVE_TARGET/" + +# Check disk space after backup +FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') +USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') +echo "๐Ÿ’พ Disk space after backup: $FREE_SPACE free ($USED_PERCENT used)" + +echo "โœ… Backup completed at $(date)" +""" + +# Auto backup script that runs during training +auto_backup_script = """#!/bin/bash + +# Auto backup script that runs during training +# Set to backup every 15 minutes to prevent disk quota issues + +BACKUP_INTERVAL=900 # 15 minutes +BACKUP_SCRIPT="/home/user/goemotions-deberta/backup_to_drive.sh" + +echo "๐Ÿ”„ Starting automatic backup service at $(date)" +echo "๐Ÿ“ Target: drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" +echo "โฑ๏ธ Backup interval: $BACKUP_INTERVAL seconds (15 minutes)" +echo "๐Ÿงน Auto-cleanup: Enabled for disk usage >85%" + +while true; do + # Run backup + echo "" + echo "๐Ÿ”„ Running scheduled backup ($(date))" + bash "$BACKUP_SCRIPT" + + # Check for disk quota issues + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + USED_NUM=${USED_PERCENT%\%} + + if [ "$USED_NUM" -gt 85 ]; then + echo "โš ๏ธ WARNING: High disk usage ($USED_PERCENT)" + echo "๐Ÿงน Cleaning old checkpoints after backup..." + # Find and list what will be removed first + echo "Finding old checkpoints to remove..." + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 + # Then remove them + find ./outputs -name "checkpoint-*" -type d | sort | head -n -5 | xargs rm -rf + + # Check space after cleanup + FREE_SPACE=$(df -h . | awk 'NR==2 {print $4}') + USED_PERCENT=$(df -h . | awk 'NR==2 {print $5}') + echo "๐Ÿ’พ Disk space after cleanup: $FREE_SPACE free ($USED_PERCENT used)" + fi + + echo "๐Ÿ’ค Next backup in $(($BACKUP_INTERVAL / 60)) minutes ($(date -d "+$BACKUP_INTERVAL seconds"))" + sleep $BACKUP_INTERVAL +done +""" + +# Function to test rclone connection +test_rclone_script = """#!/bin/bash + +# Test rclone connection to Google Drive +echo "๐Ÿ” Testing rclone connection to Google Drive..." + +# Check if rclone is installed +if ! command -v rclone &> /dev/null; then + echo "โŒ rclone not found. Please install it first." + exit 1 +fi + +# Check if drive remote exists +if ! rclone listremotes | grep -q "drive:"; then + echo "โŒ 'drive:' remote not found in rclone configuration." + echo "Please configure rclone with 'rclone config' command first." + exit 1 +fi + +# Try to list root directory +echo "๐Ÿ“ Testing access to drive: remote..." +if ! rclone lsd drive: &>/dev/null; then + echo "โŒ Cannot access 'drive:' remote. Please check your rclone configuration." + exit 1 +fi + +# Try to access or create the backup directory +TARGET_DIR="drive:00_Projects/๐ŸŽฏ TechLabs-2025/Final_Project/TRAINING/GoEmotions-DeBERTa-Backup" +echo "๐Ÿ“ Testing access to target directory: $TARGET_DIR" + +if ! rclone lsd "$TARGET_DIR" &>/dev/null; then + echo "๐Ÿ”จ Target directory doesn't exist. Creating it now..." + if ! rclone mkdir "$TARGET_DIR" &>/dev/null; then + echo "โŒ Failed to create target directory. Please check permissions and path." + exit 1 + else + echo "โœ… Target directory created successfully." + fi +else + echo "โœ… Target directory exists and is accessible." +fi + +# Try writing a test file +TEST_FILE="/tmp/rclone_test_$(date +%s).txt" +echo "This is a test file. Created at $(date)" > "$TEST_FILE" + +echo "๐Ÿ“ค Uploading test file to $TARGET_DIR..." +if ! rclone copy "$TEST_FILE" "$TARGET_DIR" &>/dev/null; then + echo "โŒ Failed to upload test file. Please check permissions." + rm -f "$TEST_FILE" + exit 1 +else + echo "โœ… Test file uploaded successfully." + + # Try to read the file back + echo "๐Ÿ“ฅ Verifying test file..." + if ! rclone ls "$TARGET_DIR/$(basename "$TEST_FILE")" &>/dev/null; then + echo "โš ๏ธ Warning: File uploaded but not immediately visible. This may be normal for Google Drive." + else + echo "โœ… File verified in target location." + # Clean up the test file + rclone delete "$TARGET_DIR/$(basename "$TEST_FILE")" &>/dev/null + fi +fi + +rm -f "$TEST_FILE" +echo "โœ… rclone connection test completed successfully!" +echo "โœ… Backup system ready to use with target: $TARGET_DIR" +""" \ No newline at end of file diff --git a/notebooks/scripts/asymmetric_loss_fix.patch b/notebooks/scripts/asymmetric_loss_fix.patch new file mode 100644 index 0000000..3bff3ea --- /dev/null +++ b/notebooks/scripts/asymmetric_loss_fix.patch @@ -0,0 +1,250 @@ +--- train_deberta_local.py.orig 2025-09-09 15:00:00.000000000 +0000 ++++ train_deberta_local.py 2025-09-09 15:10:00.000000000 +0000 +@@ -26,6 +26,9 @@ + os.environ["NCCL_TIMEOUT"] = "1800" # 30 minutes timeout (reduced from 1 hour) + os.environ["NCCL_BLOCKING_WAIT"] = "1" # Enable blocking wait + os.environ["NCCL_ASYNC_ERROR_HANDLING"] = "1" # Better error handling ++os.environ["NCCL_SOCKET_IFNAME"] = "lo" # Force localhost-only communication ++os.environ["NCCL_IB_DISABLE"] = "1" # Disable InfiniBand ++os.environ["NCCL_DEBUG"] = "INFO" # Enable debug info for troubleshooting + + from typing import List, Dict, Any + import torch +@@ -35,7 +38,8 @@ + from torch.utils.data import Dataset + from transformers import ( + AutoTokenizer, AutoModelForSequenceClassification, AutoConfig, +- Trainer, TrainingArguments, DataCollatorWithPadding ++ Trainer, TrainingArguments, DataCollatorWithPadding, ++ TrainerCallback, TrainerState, TrainerControl + ) + from sklearn.metrics import f1_score, precision_recall_fscore_support + import logging +@@ -43,6 +47,9 @@ + import random + from datetime import datetime + ++import sys ++import traceback ++import shutil + + # Set up logging + logging.basicConfig(level=logging.INFO) +@@ -123,6 +130,67 @@ + with open(self.log_file, 'a') as f: + f.write(json.dumps(log_data) + '\n') + ++# ProgressMonitorCallback to detect training stalls ++class ProgressMonitorCallback(TrainerCallback): ++ """Monitors progress and detects stalls in training""" ++ ++ def __init__(self, stall_timeout=600, disk_quota_check=True, min_free_space_gb=10): ++ """Initialize ProgressMonitor ++ ++ Args: ++ stall_timeout: Seconds without progress before considering training stalled ++ disk_quota_check: Whether to check disk space ++ min_free_space_gb: Minimum free disk space in GB before warning ++ """ ++ self.last_step = 0 ++ self.last_progress_time = time.time() ++ self.stall_timeout = stall_timeout ++ self.disk_quota_check = disk_quota_check ++ self.min_free_space_gb = min_free_space_gb ++ self.check_disk_space() ++ ++ def check_disk_space(self): ++ """Check available disk space""" ++ if not self.disk_quota_check: ++ return True ++ ++ try: ++ # Get disk usage of current directory ++ disk = shutil.disk_usage('.') ++ free_gb = disk.free / (1024 ** 3) ++ total_gb = disk.total / (1024 ** 3) ++ used_percent = (disk.used / disk.total) * 100 ++ ++ print(f"๐Ÿ’พ Disk space: {free_gb:.1f}GB free / {total_gb:.1f}GB total ({used_percent:.1f}% used)") ++ ++ if free_gb < self.min_free_space_gb: ++ print(f"โš ๏ธ LOW DISK SPACE WARNING: Only {free_gb:.1f}GB free. Training may fail with disk quota error.") ++ if used_percent > 85: ++ print("๐Ÿ”ฅ CRITICAL: Disk usage above 85%. Try to free up space immediately!") ++ return False ++ ++ return True ++ except Exception as e: ++ print(f"โš ๏ธ Error checking disk space: {e}") ++ return True ++ ++ def on_step_end(self, args, state, control, **kwargs): ++ """Called after each step""" ++ # Update progress tracker if step increased ++ if state.global_step > self.last_step: ++ self.last_step = state.global_step ++ self.last_progress_time = time.time() ++ ++ # Check if we're stalled ++ time_since_progress = time.time() - self.last_progress_time ++ if time_since_progress > self.stall_timeout: ++ print(f"โš ๏ธ WARNING: No progress for {time_since_progress:.1f} seconds (stall detected)") ++ print(f"๐Ÿ” Last progress at step {self.last_step}") ++ ++ # Check disk space on stall detection ++ if not self.check_disk_space(): ++ print("โŒ TRAINING STOPPED: Disk space critical. Clean up files before continuing.") ++ raise RuntimeError("Training stopped due to disk quota issue") ++ ++ return control + + # GoEmotions labels + EMOTION_LABELS = [ +@@ -139,7 +207,7 @@ +- def __init__(self, gamma_neg=1.0, gamma_pos=1.0, clip=0.2, eps=1e-8, disable_torch_grad_focal_loss=True): ++ def __init__(self, gamma_neg=1.0, gamma_pos=1.0, clip=0.2, eps=1e-8, disable_torch_grad_focal_loss=False): + super(AsymmetricLoss, self).__init__() + self.gamma_neg = gamma_neg + self.gamma_pos = gamma_pos +@@ -173,12 +241,7 @@ + if self.gamma_neg > 0 or self.gamma_pos > 0: + # FIXED: Remove no_grad for full differentiability (HF docs compliant) + if self.disable_torch_grad_focal_loss: +- with torch.no_grad(): +- pt0 = xs_pos * y +- pt1 = xs_neg * (1 - y) # pt = p if t > 0 else 1-p +- pt = pt0 + pt1 +- one_sided_gamma = self.gamma_pos * y + self.gamma_neg * (1 - y) +- one_sided_w = torch.pow(1 - pt, one_sided_gamma) ++ # WARNING: This code path has gradient disconnection - never use this! + loss = loss * one_sided_w + else: + pt0 = xs_pos * y +@@ -247,7 +310,7 @@ + """ + def __init__(self, loss_combination_ratio=0.7, *args, **kwargs): + super().__init__(*args, **kwargs) +- self.asymmetric_loss = AsymmetricLoss(gamma_neg=1.0, gamma_pos=1.0, clip=0.2, disable_torch_grad_focal_loss=True) ++ self.asymmetric_loss = AsymmetricLoss(gamma_neg=1.0, gamma_pos=1.0, clip=0.2, disable_torch_grad_focal_loss=False) + self.focal_loss = FocalLoss(alpha=0.25, gamma=2.0) + self.loss_combination_ratio = loss_combination_ratio + +@@ -384,7 +447,7 @@ + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) +- self.asymmetric_loss = AsymmetricLoss(gamma_neg=1.0, gamma_pos=1.0, clip=0.2, disable_torch_grad_focal_loss=True) ++ self.asymmetric_loss = AsymmetricLoss(gamma_neg=1.0, gamma_pos=1.0, clip=0.2, disable_torch_grad_focal_loss=False) + + def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None): + """ +@@ -680,6 +743,42 @@ + return None, None + + ++def check_system_status(): ++ """Check system status and resources""" ++ print("\n๐Ÿ” Checking system status...") ++ ++ # Check disk space ++ try: ++ disk = shutil.disk_usage('.') ++ free_gb = disk.free / (1024 ** 3) ++ total_gb = disk.total / (1024 ** 3) ++ used_percent = (disk.used / disk.total) * 100 ++ print(f"๐Ÿ’พ Disk space: {free_gb:.1f}GB free / {total_gb:.1f}GB total ({used_percent:.1f}% used)") ++ ++ if free_gb < 10: ++ print(f"โš ๏ธ LOW DISK SPACE WARNING: Only {free_gb:.1f}GB free") ++ if used_percent > 85: ++ print("๐Ÿ”ฅ CRITICAL: Disk usage above 85%. Training may fail!") ++ except Exception as e: ++ print(f"โš ๏ธ Error checking disk space: {e}") ++ ++ # Check GPU status ++ try: ++ if torch.cuda.is_available(): ++ for i in range(torch.cuda.device_count()): ++ print(f"GPU {i}: {torch.cuda.get_device_name(i)}") ++ print(f" Memory: {torch.cuda.memory_allocated(i)/1024**2:.1f}MB allocated, " ++ f"{torch.cuda.memory_reserved(i)/1024**2:.1f}MB reserved") ++ else: ++ print("โŒ No GPUs available") ++ except Exception as e: ++ print(f"โš ๏ธ Error checking GPU status: {e}") ++ ++ print("โœ… System check complete\n") ++ ++ ++def handle_exception(e, scientific_logger=None): ++ """Handle exceptions with comprehensive error logging""" ++ error_msg = f"โŒ ERROR: {str(e)}" ++ print(error_msg) ++ print("Stack trace:") ++ traceback_str = traceback.format_exc() ++ print(traceback_str) ++ ++ # Log to scientific logger if available ++ if scientific_logger: ++ error_log = { ++ "timestamp": datetime.now().isoformat(), ++ "error": str(e), ++ "traceback": traceback_str, ++ } ++ scientific_logger._write_log(error_log) ++ ++ return 1 ++ ++ + def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", type=str, default="./outputs/deberta") +@@ -717,10 +816,13 @@ + print(f"๐Ÿ“Š Dataset: GoEmotions (from local cache)") + print(f"๐Ÿ”ฌ Scientific logging: ENABLED") + ++ # Check system resources ++ check_system_status() ++ + # Create output directory +- os.makedirs(args.output_dir, exist_ok=True) ++ try: ++ os.makedirs(args.output_dir, exist_ok=True) ++ except Exception as e: ++ print(f"โŒ ERROR: Could not create output directory: {e}") ++ print("๐Ÿ’ก TIP: Check disk quota or remove old checkpoints") ++ return 1 + + # Initialize scientific logger + scientific_logger = ScientificLogger(args.output_dir) +@@ -813,6 +915,9 @@ + compute_metrics=compute_comprehensive_metrics, + ) + ++ # Add progress monitoring callback ++ progress_monitor = ProgressMonitorCallback(stall_timeout=600) # 10 minutes timeout ++ trainer.add_callback(progress_monitor) + + # Train + print("๐Ÿš€ Starting training...") +- trainer.train() ++ try: ++ trainer.train() ++ except Exception as e: ++ handle_exception(e, scientific_logger) ++ return 1 + + # Save final model + trainer.save_model() +@@ -904,4 +1009,10 @@ + + + if __name__ == "__main__": +- main() ++ try: ++ main() ++ except Exception as e: ++ print(f"โŒ UNHANDLED ERROR: {str(e)}") ++ traceback.print_exc() ++ sys.exit(1) ++ \ No newline at end of file diff --git a/notebooks/scripts/progress_monitor_callback.py b/notebooks/scripts/progress_monitor_callback.py new file mode 100644 index 0000000..4827f15 --- /dev/null +++ b/notebooks/scripts/progress_monitor_callback.py @@ -0,0 +1,135 @@ +""" +Progress Monitor Callback for Transformers training +Detects stalls in training and checks disk space +""" + +import time +import shutil +import traceback +from datetime import datetime +from transformers import TrainerCallback, TrainerState, TrainerControl + +class ProgressMonitorCallback(TrainerCallback): + """Monitors progress and detects stalls in training""" + + def __init__(self, stall_timeout=600, disk_quota_check=True, min_free_space_gb=10): + """Initialize ProgressMonitor + + Args: + stall_timeout: Seconds without progress before considering training stalled + disk_quota_check: Whether to check disk space + min_free_space_gb: Minimum free disk space in GB before warning + """ + self.last_step = 0 + self.last_progress_time = time.time() + self.stall_timeout = stall_timeout + self.disk_quota_check = disk_quota_check + self.min_free_space_gb = min_free_space_gb + self.check_disk_space() + + def check_disk_space(self): + """Check available disk space""" + if not self.disk_quota_check: + return True + + try: + # Get disk usage of current directory + disk = shutil.disk_usage('.') + free_gb = disk.free / (1024 ** 3) + total_gb = disk.total / (1024 ** 3) + used_percent = (disk.used / disk.total) * 100 + + print(f"๐Ÿ’พ Disk space: {free_gb:.1f}GB free / {total_gb:.1f}GB total ({used_percent:.1f}% used)") + + if free_gb < self.min_free_space_gb: + print(f"โš ๏ธ LOW DISK SPACE WARNING: Only {free_gb:.1f}GB free. Training may fail with disk quota error.") + if used_percent > 85: + print("๐Ÿ”ฅ CRITICAL: Disk usage above 85%. Try to free up space immediately!") + return False + + return True + except Exception as e: + print(f"โš ๏ธ Error checking disk space: {e}") + return True + + def on_step_end(self, args, state, control, **kwargs): + """Called after each step""" + # Update progress tracker if step increased + if state.global_step > self.last_step: + self.last_step = state.global_step + self.last_progress_time = time.time() + + # Check if we're stalled + time_since_progress = time.time() - self.last_progress_time + if time_since_progress > self.stall_timeout: + print(f"โš ๏ธ WARNING: No progress for {time_since_progress:.1f} seconds (stall detected)") + print(f"๐Ÿ” Last progress at step {self.last_step}") + + # Check disk space on stall detection + if not self.check_disk_space(): + print("โŒ TRAINING STOPPED: Disk space critical. Clean up files before continuing.") + # We cannot set control.should_training_stop because it's read-only + # Instead we'll raise an exception to stop training + raise RuntimeError("Training stopped due to disk quota issue") + + # Check disk space every 100 steps + if state.global_step % 100 == 0 and self.disk_quota_check: + self.check_disk_space() + + return control + +def check_system_status(): + """Check system status and resources""" + print("\n๐Ÿ” Checking system status...") + + # Check disk space + try: + disk = shutil.disk_usage('.') + free_gb = disk.free / (1024 ** 3) + total_gb = disk.total / (1024 ** 3) + used_percent = (disk.used / disk.total) * 100 + print(f"๐Ÿ’พ Disk space: {free_gb:.1f}GB free / {total_gb:.1f}GB total ({used_percent:.1f}% used)") + + if free_gb < 10: + print(f"โš ๏ธ LOW DISK SPACE WARNING: Only {free_gb:.1f}GB free") + if used_percent > 85: + print("๐Ÿ”ฅ CRITICAL: Disk usage above 85%. Training may fail!") + except Exception as e: + print(f"โš ๏ธ Error checking disk space: {e}") + + # Check GPU status + try: + import torch + if torch.cuda.is_available(): + for i in range(torch.cuda.device_count()): + print(f"GPU {i}: {torch.cuda.get_device_name(i)}") + print(f" Memory: {torch.cuda.memory_allocated(i)/1024**2:.1f}MB allocated, " + f"{torch.cuda.memory_reserved(i)/1024**2:.1f}MB reserved") + else: + print("โŒ No GPUs available") + except Exception as e: + print(f"โš ๏ธ Error checking GPU status: {e}") + + print("โœ… System check complete\n") + +def handle_exception(e, scientific_logger=None): + """Handle exceptions with comprehensive error logging""" + error_msg = f"โŒ ERROR: {str(e)}" + print(error_msg) + print("Stack trace:") + traceback_str = traceback.format_exc() + print(traceback_str) + + # Log to scientific logger if available + if scientific_logger: + error_log = { + "timestamp": datetime.now().isoformat(), + "error": str(e), + "traceback": traceback_str, + } + try: + scientific_logger._write_log(error_log) + except: + pass + + return 1 \ No newline at end of file