upload_to_lambda.sh

#!/bin/bash
# Package and upload training data + scripts to Lambda GPU instance
#
# Usage:
#   bash hackathon/upload_to_lambda.sh <LAMBDA_IP>
#
# Example:
#   bash hackathon/upload_to_lambda.sh 192.168.1.100

set -euo pipefail

if [ $# -lt 1 ]; then
    echo "Usage: $0 <LAMBDA_IP_OR_HOSTNAME>"
    echo "Example: $0 ubuntu@104.171.203.50"
    exit 1
fi

REMOTE="$1"
# Add ubuntu@ prefix if not present
if [[ "$REMOTE" != *@* ]]; then
    REMOTE="ubuntu@${REMOTE}"
fi

PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"

echo "=== Uploading to Lambda: ${REMOTE} ==="
echo "Project root: ${PROJECT_ROOT}"
echo ""

# Create remote directories
echo "Creating remote directories..."
ssh "$REMOTE" "mkdir -p ~/hackathon ~/data/training ~/data/cpa-far ~/data/eval"

# Upload hackathon scripts
echo "Uploading hackathon scripts..."
scp -r "${PROJECT_ROOT}/hackathon/"*.py "${PROJECT_ROOT}/hackathon/"*.sh "${PROJECT_ROOT}/hackathon/"*.txt "$REMOTE:~/hackathon/"

# Upload training data
echo "Uploading SFT training data (47 MB + 4.5 MB)..."
scp "${PROJECT_ROOT}/data/training/e2_sft_train.jsonl" "$REMOTE:~/data/training/"
scp "${PROJECT_ROOT}/data/training/e2_sft_val.jsonl" "$REMOTE:~/data/training/"

# Upload CPA FAR data
echo "Uploading CPA FAR questions (33 MB)..."
scp -r "${PROJECT_ROOT}/data/cpa-far/" "$REMOTE:~/data/cpa-far/"

# Upload eval data
echo "Uploading eval benchmarks..."
scp "${PROJECT_ROOT}/data/eval/"*.jsonl "$REMOTE:~/data/eval/"

# Upload curriculum data (optional, 64 MB)
echo "Uploading curriculum data (64 MB)..."
scp "${PROJECT_ROOT}/data/training/e2_curriculum_v1.jsonl" "$REMOTE:~/data/training/" || echo "Skipped curriculum (optional)"

echo ""
echo "=== Upload complete! ==="
echo ""
echo "Total uploaded: ~150 MB"
echo ""
echo "Next steps:"
echo "  ssh $REMOTE"
echo "  bash ~/hackathon/setup_lambda.sh"
echo ""
echo "Or run everything in one shot:"
echo "  ssh $REMOTE 'bash ~/hackathon/setup_lambda.sh && source ~/venv/bin/activate && python ~/hackathon/train_qlora.py --run journal'"