├── __init__.py # Python package initialization file
├── main.py # Parent script to execute all flow
├── pyproject.toml # Project configuration for Python tools
├── uv.lock # Dependency lock file for uv package manager
├── .gitignore # Specifies intentionally untracked files to ignore
├── .pre-commit-config.yaml # Configuration for pre-commit hooks
├── .python-version # Python version specification
├── README.md # Project overview and documentation
|
├── data/ # Data directory for datasets with 100 random questions
│ ├── ai2_arc_questions.json # AI2 Reasoning Challenge dataset
│ ├── mathqa_questions.json # Math QA questions dataset
│ ├── medmcqa_questions.json # Medical MCQ questions dataset
│ ├── mmlu_questions.json # MMLU benchmark questions
│ ├── pubmedqa_questions.json # PubMed QA dataset
│ └── tau_cqa_questions.json # Tau Clinical QA dataset
│
├── rephrased_data/ # Data directory for datasets with the rephrased questions from data folder.
│ └── ...
│
├── docs/ # Documentation files
│ ├── CodePractices.md # Code practices and structure documentation
│ ├── datasets.md # Dataset documentation
│ ├── Install.md # Installation instructions
│ └── models.md # Model documentation
│
├── plots/ # Directory for visualization plots
│ ├── individual/ # Individual plots directory
│ └── comparison/ # Comparison plots directory
|
├── results/ # Results output directory
│ └── ... # Result files for LLMs answers on questions
│
|
├── analysis_output_comparison_betabinom/ # Bayesian results for kappa, mu, gamma, svg images
│ └── ...
│
├── csv/ # Tables for simple statistics score (mean, stdev) for 4 datasets on all models
│ └── ...
|
└── src/ # Source code directory
├── __init__.py # Package initialization file
├── constants.py # Project constants and configurations
├── custom_logger.py # Custom logging setup
├── example_prompts.py # Example prompts for different datasets
├── llm_utils.py # LLM API interaction utilities
├── parser.py # Command-line argument parser
├── plots.py # Visualization and plotting utilities
├── questions.py # Dataset loading and question processing
├── Bayesian_Benchmark_Inter_Intra.py # Bayesian Beta-Binomial framework processing
├── Plot_bt_distance.py # Plotting Bhattacharyya Distances
└── utils.py # General utility functions and evaluation logic