pyproject.toml

[project]
name = "pyrit"
version = "1.2.0.dev0"
description = "The Python Risk Identification Tool for LLMs (PyRIT) is a library used to assess the robustness of LLMs"
authors = [
    { name = "Microsoft AI Red Team", email = "airedteam@microsoft.com" },
]
readme = "README.md"
license = "MIT"
keywords = [
    "llm",
    "ai-safety",
    "ai-security",
    "ai-red-team",
    "ai-red-teaming",
    "ai-robustness",
    "ai-robustness-testing",
    "ai-risk-assessment",
]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
]
requires-python = ">=3.11, <3.15"
dependencies = [
    "aiofiles>=24,<26",
    "aioodbc>=0.5.0",
    "aiosqlite>=0.21.0",
    "alembic>=1.16.0",
    "appdirs>=1.4.0",
    "art>=6.5.0",
    "av>=14.0.0",
    "azure-core>=1.38.0",
    "azure-identity>=1.19.0",
    "azure-keyvault-secrets>=4.9.0",
    "azure-ai-contentsafety>=1.0.0",
    "azure-storage-blob>=12.19.0",
    "base2048>=0.1.3",
    "colorama>=0.4.6",
    "confusables>=1.2.0",
    "confusable-homoglyphs>=3.3.1",
    "ecoji>=0.1.1",
    "datasets>=4.8.0",
    "fastapi>=0.133.0",
    "httpx[http2]>=0.27.2",
    "httpx2>=2.5.0,<3",
    "jinja2>=3.1.6",
    "jsonschema>=4.25.1",
    "markdown-it-py>=4.0.0,<5",
    "mcp>=2.2,<3",
    "numpy>=1.26.0; python_version < '3.14'",
    "numpy>=2.3.0; python_version >= '3.14'",
    "openai>=3,<4",
    "openpyxl>=3.1.5",
    "opentelemetry-sdk>=1.36,<2",
    "pillow>=12.2.0",
    "pydantic>=2.12.0",
    "PyJWT[crypto]>=2.15.0",
    "pyodbc>=5.1.0",
    "pypdf>=6.10.2",
    "pypinyin>=0.55.0",
    "python-docx>=1.1.0",
    "python-dotenv>=1.2.2",
    "reportlab>=4.4.4",
    "regex>=2024.11.6",
    "segno>=1.6.6",
    "scipy>=1.15.3",
    "SQLAlchemy[asyncio]>=2.0.41",
    "starlette>=1.0.1",
    "termcolor>=2.4.0",
    "tenacity>=9.1.2",
    "tinytag>=2.2.1",
    "tqdm>=4.67.1",
    "transformers>=5.0.0rc3",
    "treelib>=1.7.1",
    "uvicorn[standard]>=0.32.0",
    "websockets>=14.0",
    "wcwidth>=0.2.14",
]

[dependency-groups]
dev = [
    "feedgen>=1.0.0",
    "griffe>=2.0.0",
    "ipykernel>=6.29.5",
    "jupyter>=1.1.1",
    "jupyter-book>=2.0.0",
    "jupytext>=1.17.1",
    "matplotlib>=3.10.0",
    "ty>=0.0.32",
    "pandas>=2.2.0",
    "pre-commit>=4.2.0",
    "pre-commit-uv>=4.3.0",
    # uv.lock selects the hook installer and venv-local uv version.
    # Project CI reads that same version with setup-uv's version-file input.
    "uv>=0.9.17",
    "pytest>=9.0.3",
    "pytest-rerunfailures>=15.1",
    "pytest-asyncio>=1.4.0",
    "diff-cover>=9.0.0",
    "pytest-cov>=6.1.1",
    "pytest-timeout>=2.4.0",
    "pytest-xdist>=3.6.1",
    "respx>=0.22.0",
    "sse-starlette>=3.0.0",
    "ruff>=0.14.4",
    "setuptools>=64.0.0",
    "types-aiofiles>=24.1.0",
    "types-PyYAML>=6.0.12.20250516",
    "types-requests>=2.31.0.20250515",
]

[project.optional-dependencies]
# always make sure the individual ones are in sync with the all group
huggingface = [
    "sentencepiece>=0.2.0",
    "torch>=2.7.0",
]
gcg = [
    "accelerate>=1.7.0",
    "azure-ai-ml>=1.32.0",
    # pyarrow is a transitive dep of `datasets`. With the gcg extra installed,
    # the resolver picks a pyarrow version that lacks cp314 wheels and fails
    # to build from source on Python 3.14; pin to a version that ships them.
    "pyarrow>=22.0.0; python_version >= '3.14'",
    "sentencepiece>=0.2.0",
    "torch>=2.7.0",
]
playwright = [
    "flask>=3.1.3",
    "ollama>=0.5.1",
    "playwright>=1.49.0",
]
fairness_bias = [
    "spacy>=3.8.13,!=3.8.14,!=3.8.15",  # 3.8.14-3.8.15 missing cp314 wheels
]
opencv = [
    "opencv-python>=4.11.0.86",
]

speech = [
    "azure-cognitiveservices-speech>=1.44.0",
]

litellm = [
    "litellm>=1.104.2",
]

github-copilot = [
    # Keep the last known working session API until the target is migrated (#3623).
    "github-copilot-sdk>=1.0.11,<1.0.23",
]
a2a = [
    "a2a-sdk>=1.1.5,<2",
]

# all includes all functional dependencies excluding the ones from the "dev" dependency group
all = [
    "a2a-sdk>=1.1.5,<2",
    "accelerate>=1.7.0",
    "azure-ai-ml>=1.32.0",
    "azure-cognitiveservices-speech>=1.44.0",
    "flask>=3.1.3",
    "github-copilot-sdk>=1.0.11,<1.0.23",
    "ipykernel>=6.29.5",
    "jupyter>=1.1.1",
    "litellm>=1.104.2",
    "ollama>=0.5.1",
    "opencv-python>=4.11.0.86",
    "playwright>=1.49.0",
    "pyarrow>=22.0.0; python_version >= '3.14'",
    "sentencepiece>=0.2.0",
    "spacy>=3.8.13,!=3.8.14,!=3.8.15",  # 3.8.14-3.8.15 missing cp314 wheels
    "torch>=2.7.0",
]

[project.scripts]
pyrit_backend = "pyrit.backend.pyrit_backend:main"
pyrit_scan = "pyrit.cli.pyrit_scan:main"
pyrit_shell = "pyrit.cli.pyrit_shell:main"

[tool.pytest.ini_options]
addopts = [
    "--import-mode=importlib",
]
pythonpath = [".", "tests"]
asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
filterwarnings = [
    # Keep the in-repo suite on the scorable/expectation contract. Tests that cover the
    # shim itself opt back in with pytest.warns.
    "error:Scorer\\.score_async:DeprecationWarning",
    # Rates come from pyrit.analytics; tests that cover the deprecated method opt back in with pytest.warns.
    "error:ScenarioResult\\.objective_achieved_rate:DeprecationWarning",
    # Ordinary callers must use async memory; compatibility tests capture the warnings.
    "error:MemoryInterface\\.:DeprecationWarning",
    "error:MessageScorableResolver\\.resolve:DeprecationWarning",
    # True/false verdicts are set through Score.set_value; compatibility tests capture the warning.
    "error:Assigning Score\\.score_value:DeprecationWarning",
]

[tool.ty]
[tool.ty.environment]
extra-paths = ["tests"]

[tool.ty.rules]
all = "error"
# Most rules under `all = "error"` are already clean for pyrit/. A few remain
# suppressed below; the remaining finding counts are noted so they can be
# re-enabled as the code is cleaned up.
# missing-type-argument is clean under numpy>=2.4 (Python 3.14) but flags 38
# bare `np.ndarray` generics under numpy 2.2.6 -- the version the lock pins for
# Python 3.11-3.13, i.e. CI's 3.11 -- across 5 files (analytics/conversation_analytics,
# converter/transparency_attack_converter, score/scorer_evaluation/*).
# Annotating those generics is real code work, so keep the rule deferred until
# that numpy-generics debt is addressed.
missing-type-argument = "ignore"
unsound-assignment = "ignore"
# unused-type-ignore-comment is environment-dependent: a `# type: ignore`
# on an optional-dependency import (torch, playwright, etc.) is "used" when the
# extra is not installed but "unused" once it is. CI installs `--extra all`, so
# enabling this rule flags such directives across several unrelated modules
# (auth/copilot_authenticator, executor/benchmark/fairness_bias,
# prompt_target/playwright_*; the incremental PR check only sees changed files,
# but the on-main `--all-files` run would fail). Cleanly enabling it requires
# refactoring those optional imports repo-wide, so keep the rule deferred.
unused-type-ignore-comment = "ignore"
# ty 0.0.43 (bumped from 0.0.32 in #1919) activated this rule under
# [tool.ty.rules] all = "error". Adopting @override across pyrit/ (~695 methods)
# would require the typing_extensions backport on Python 3.11
# (typing.override is 3.12+), which adds a direct dep and ongoing import-path
# friction. Suppressing the rule keeps the codebase consistent for now.
missing-override-decorator = "ignore"
# Intentional truthiness checks also reject empty values; keep that idiom without redundant bool() wrappers.
truthiness-test-of-none-union = "ignore"

[tool.ty.analysis]
respect-type-ignore-comments = true
# Treat colorama as Any — it has no type stubs and uses a dynamic pattern
# (class attrs are int, __init__ converts to str via setattr) that ty can't follow.
# cv2 (opencv) and azure.cognitiveservices.speech are optional extras that are not
# installed in the default environment, so their lazy imports are unresolvable; treat
# them as Any rather than erroring on the guarded ``import`` statements.
replace-imports-with-any = [
    "colorama.**",
    "cv2.**",
    "azure.cognitiveservices.speech.**",
]

[tool.ty.src]
exclude = ["doc/code/"]

# Relax type checking for HuggingFace module
[[tool.ty.overrides]]
include = ["pyrit/prompt_target/hugging_face/**"]
[tool.ty.overrides.rules]
invalid-argument-type = "ignore"

# Historical Alembic revisions are immutable; retain their defensive rowcount checks.
[[tool.ty.overrides]]
include = ["pyrit/memory/alembic/versions/1b3d5f7a9c2e_persist_scored_expectation.py"]
[tool.ty.overrides.rules]
redundant-condition-strict = "ignore"

# Unit tests intentionally exercise runtime validation failures, mutable test doubles,
# and optional fields after fixtures have populated them. Keep library code strict while
# avoiding hundreds of per-assertion ignores in tests.
[[tool.ty.overrides]]
include = ["tests/unit/**"]
[tool.ty.overrides.rules]
call-non-callable = "ignore"
invalid-assignment = "ignore"
invalid-await = "ignore"
invalid-parameter-default = "ignore"
invalid-type-arguments = "ignore"
invalid-yield = "ignore"
missing-argument = "ignore"
not-subscriptable = "ignore"
unknown-argument = "ignore"
unsupported-operator = "ignore"
# Re-enabled repo-wide so pyrit/ is checked (production is clean) but still
# relaxed here: tests read private/mock attributes and subclass library types
# with loosened signatures / runtime-validation-failure arguments.
invalid-argument-type = "ignore"
invalid-method-override = "ignore"
no-matching-overload = "ignore"
unresolved-attribute = "ignore"

[tool.uv]
constraint-dependencies = [
    "aiohttp>=3.13.4",
    "GitPython>=3.1.50",
    "jupyter-server>=2.18.0",
    "jupyterlab>=4.6.4",
    "lxml>=6.1.0",
    "Mako>=1.3.11",
    "mistune>=3.2.1",
    "nbconvert>=7.17.1",
    "notebook>=7.6.3",
    "orjson>=3.11.6",
    "pyasn1>=0.6.3",
    "cryptography>=46.0.7",
    "Pygments>=2.20.0",
    "PyJWT>=2.15.0",
    "requests>=2.33.0",
    "tornado>=6.5.9",
    "urllib3>=2.8.0",
    "virtualenv>=21.7.13",
    "werkzeug>=3.1.6",
]

[build-system]
requires = ["setuptools>=64.0.0", "wheel>=0.46.2"]
build-backend = "build_scripts.build_backend"
backend-path = ["."]

[tool.setuptools.packages.find]
include = ["pyrit", "pyrit.*"]

[tool.setuptools]
license-files = [
    "LICENSE",
    "LICENSES/Apache-2.0.txt",
    "LICENSES/PromptInject-MIT.txt",
    "THIRD_PARTY_NOTICES.txt",
    "third_party/garak-provenance.json",
    "third_party/promptinject-provenance.json",
]

[tool.setuptools.package-data]
pyrit = [
    "_compatibility.json",
    "backend/frontend/**/*",
    "memory/alembic/**/*",
    "memory/alembic.ini",
    "py.typed"
]

[tool.jupytext]
formats = "ipynb,py:percent"

[tool.ruff]
line-length = 120
target-version = "py311"

[tool.ruff.lint]
preview = true
fixable = [
    "A",
    "B",
    "C",
    "D",
    "E",
    "F",
    "G",
    "I",
    "N",
    "Q",
    "S",
    "T",
    "W",
    "ANN",
    "ARG",
    "BLE",
    "COM",
    "DJ",
    "DTZ",
    "EM",
    "ERA",
    "EXE",
    "FBT",
    "ICN",
    "INP",
    "ISC",
    "NPY",
    "PD",
    "PERF",
    "PGH",
    "PIE",
    "PL",
    "PT",
    "PTH",
    "PYI",
    "RET",
    "RSE",
    "RUF",
    "SIM",
    "SLF",
    "TCH",
    "TID",
    "TRY",
    "UP",
    "YTT",
]
select = [
    "A",  # https://docs.astral.sh/ruff/rules/#flake8-builtins-a
    "B",  # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
    "C4",  # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
    "CPY001",  # missing-copyright-notice
    # "D" + preview=true enables D421 (property-docstring-starts-with-verb):
    # property docstrings must be noun phrases ("""The display name."""), not
    # verb phrases ("""Return the display name."""). Keep "D" selected, not ignored.
    "D",  # https://docs.astral.sh/ruff/rules/#pydocstyle-d
    "DOC",  # https://docs.astral.sh/ruff/rules/#pydoclint-doc
    "DTZ",  # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz
    "E",  # https://docs.astral.sh/ruff/rules/#pycodestyle-e
    "ERA",  # https://docs.astral.sh/ruff/rules/#eradicate-era
    "EXE",  # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
    "F401",  # unused-import
    "FA",  # https://docs.astral.sh/ruff/rules/#flake8-future-annotations-fa
    "FLY",  # https://docs.astral.sh/ruff/rules/#flynt-fly
    "I",  # isort
    "ICN",  # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
    "ISC",  # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
    "LOG",  # https://docs.astral.sh/ruff/rules/#flake8-logging-log
    "N",  # https://docs.astral.sh/ruff/rules/#pep8-naming-n
    "PERF",  # https://docs.astral.sh/ruff/rules/#perflint-perf
    "PGH",  # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh
    "PIE",  # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
    "PLE",  # https://docs.astral.sh/ruff/rules/#pylint-ple
    "Q",  # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
    "RET",  # https://docs.astral.sh/ruff/rules/#flake8-return-ret
    "RSE",  # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
    "SIM",  # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
    "SLOT",  # https://docs.astral.sh/ruff/rules/#flake8-slots-slot
    "T10",  # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10
    "TCH",  # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch
    "TID",  # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
    "UP",  # https://docs.astral.sh/ruff/rules/#pyupgrade-up
    "W",  # https://docs.astral.sh/ruff/rules/#pycodestyle-w
    "YTT",  # https://docs.astral.sh/ruff/rules/#flake8-2020-ytt
    "ANN",  # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
]
ignore = [
    "ANN401",  # any-type (too strict — many legitimate uses of Any)
    "B903",  # class-as-data-structure (test helper classes use @apply_defaults pattern)
    "D100",  # Missing docstring in public module
    "D200",  # One-line docstring should fit on one line
    "D205",  # 1 blank line required between summary line and description
    "D212",  # Multi-line docstring summary should start at the first line
    "D301",  # Use r""" if any backslashes in a docstring
    "DOC502",  # Raised exception is not explicitly raised
    "PERF203",  # try-except-in-loop (intentional per-item error handling)
    "SIM117",  # multiple-with-statements (combining often exceeds line length)
    "UP042",  # replace-str-enum: StrEnum changes str(member) from "Class.MEMBER" to member.value
]
extend-select = [
    "D204",  # 1 blank line required after class docstring
    "D213",  # Multi-line docstring summary should start at the second line
    "D401",  # First line of docstring should be in imperative mood
    "D404",  # First word of the docstring should not be "This"
]

[tool.ruff.lint.isort]
known-first-party = ["pyrit"]

[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"

[tool.ruff.lint.flake8-type-checking]
# Pydantic resolves model field annotations at runtime (the models use
# ``from __future__ import annotations``), so imports used only as field types
# must stay at module level. Ruff auto-detects direct ``pydantic.BaseModel``
# subclasses, but not models whose base is an intermediate PyRIT class, so we
# enumerate those bases here to keep their field imports runtime-evaluated.
runtime-evaluated-base-classes = [
    "pyrit.models.StrategyResult",
    "pyrit.models.results.strategy_result.StrategyResult",
]

[tool.ruff.lint.flake8-copyright]
min-file-size = 1
notice-rgx = "Copyright \\(c\\) Microsoft Corporation\\.\\s*\\n.*Licensed under the MIT license"

[tool.ruff.lint.per-file-ignores]
# Ignore D and DOC rules everywhere except for the pyrit/ directory
"!pyrit/**.py" = ["D", "DOC"]
# Ignore copyright, import-location (E402), line length (E501), type-checking, and blanket type-ignore (PGH003) rules in doc/ directory
# E402: notebooks have imports in code cells, not at top of file
# E501: markdown comments in notebooks can exceed line length
# PGH003: blanket type: ignore in notebooks (not type-checked)
# A: notebook code commonly uses builtins like id, dir, display
# ERA001: jupytext markdown cells look like commented-out code
# LOG: notebooks use root logger for convenience
# PLE1142: top-level await is valid in Jupyter notebooks
"doc/**" = ["ANN", "CPY001", "TCH", "E402", "E501", "PGH003", "A", "ERA001", "LOG", "PLE1142"]
# Backend API routes raise HTTPException handled by FastAPI, not true exceptions
"pyrit/backend/**/*.py" = ["DOC501", "B008"]
"pyrit/__init__.py" = ["D104"]
# Allow broad pytest.raises(Exception) in tests
"tests/**/*.py" = ["ANN", "B017"]
# Infrastructure/utility scripts — not part of the library API
"build_scripts/**/*.py" = ["ANN"]
"docker/**/*.py" = ["ANN"]
"frontend/dev.py" = ["ANN"]

[tool.ruff.lint.pydocstyle]
convention = "google"