diff --git a/.github/workflows/mononoke_linux.yml b/.github/workflows/mononoke_linux.yml index 16b09a2be54f2..89bff42ba2d80 100644 --- a/.github/workflows/mononoke_linux.yml +++ b/.github/workflows/mononoke_linux.yml @@ -3,7 +3,12 @@ name: Mononoke Linux on: - workflow_dispatch + push: + branches: + - main + pull_request: + branches: + - main permissions: contents: read # to fetch code (actions/checkout) @@ -26,8 +31,8 @@ jobs: - id: paths name: Query paths run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. mononoke >> "$GITHUB_OUTPUT" - - name: Install Rust Stable - uses: dtolnay/rust-toolchain@stable + - name: Install Rust 1.91 + uses: dtolnay/rust-toolchain@1.91 - name: Fetch xxhash if: ${{ steps.paths.outputs.xxhash_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash @@ -133,7 +138,7 @@ jobs: key: ${{ steps.paths.outputs.xxhash_CACHE_KEY }}-install - name: Build xxhash if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests xxhash - name: Save xxhash to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} @@ -149,7 +154,7 @@ jobs: key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - name: Build ninja if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests ninja - name: Save ninja to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} @@ -165,7 +170,7 @@ jobs: key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - name: Build cmake if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests cmake - name: Save cmake to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} @@ -181,7 +186,7 @@ jobs: key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install - name: Build fmt if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fmt - name: Save fmt to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} @@ -197,7 +202,7 @@ jobs: key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install - name: Build googletest if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests googletest - name: Save googletest to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} @@ -213,7 +218,7 @@ jobs: key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install - name: Build zstd if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests zstd - name: Save zstd to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} @@ -229,7 +234,7 @@ jobs: key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install - name: Build boost if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests boost - name: Save boost to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} @@ -245,7 +250,7 @@ jobs: key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install - name: Build double-conversion if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests double-conversion - name: Save double-conversion to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} @@ -261,7 +266,7 @@ jobs: key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install - name: Build fast_float if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fast_float + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fast_float - name: Save fast_float to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} @@ -277,7 +282,7 @@ jobs: key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install - name: Build gflags if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests gflags - name: Save gflags to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} @@ -293,7 +298,7 @@ jobs: key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install - name: Build glog if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests glog - name: Save glog to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} @@ -309,7 +314,7 @@ jobs: key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install - name: Build libdwarf if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libdwarf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libdwarf - name: Save libdwarf to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} @@ -325,7 +330,7 @@ jobs: key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install - name: Build libevent if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libevent - name: Save libevent to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} @@ -341,7 +346,7 @@ jobs: key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install - name: Build lz4 if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests lz4 - name: Save lz4 to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} @@ -357,7 +362,7 @@ jobs: key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install - name: Build snappy if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests snappy - name: Save snappy to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} @@ -373,7 +378,7 @@ jobs: key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install - name: Build zlib if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests zlib - name: Save zlib to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} @@ -389,7 +394,7 @@ jobs: key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install - name: Build openssl if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests openssl - name: Save openssl to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} @@ -405,7 +410,7 @@ jobs: key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install - name: Build liboqs if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests liboqs + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests liboqs - name: Save liboqs to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} @@ -421,7 +426,7 @@ jobs: key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - name: Build autoconf if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests autoconf - name: Save autoconf to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} @@ -437,7 +442,7 @@ jobs: key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - name: Build automake if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests automake - name: Save automake to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} @@ -453,7 +458,7 @@ jobs: key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - name: Build libtool if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libtool - name: Save libtool to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} @@ -469,7 +474,7 @@ jobs: key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install - name: Build libsodium if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libsodium - name: Save libsodium to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} @@ -485,7 +490,7 @@ jobs: key: ${{ steps.paths.outputs.libiberty_CACHE_KEY }}-install - name: Build libiberty if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libiberty + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libiberty - name: Save libiberty to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }} @@ -501,7 +506,7 @@ jobs: key: ${{ steps.paths.outputs.libunwind_CACHE_KEY }}-install - name: Build libunwind if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libunwind + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libunwind - name: Save libunwind to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }} @@ -517,7 +522,7 @@ jobs: key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install - name: Build xz if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests xz - name: Save xz to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} @@ -533,7 +538,7 @@ jobs: key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install - name: Build folly if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests folly - name: Save folly to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} @@ -549,7 +554,7 @@ jobs: key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install - name: Build fizz if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fizz - name: Save fizz to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} @@ -565,7 +570,7 @@ jobs: key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install - name: Build mvfst if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests mvfst - name: Save mvfst to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} @@ -581,7 +586,7 @@ jobs: key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install - name: Build wangle if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests wangle - name: Save wangle to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} @@ -597,7 +602,7 @@ jobs: key: ${{ steps.paths.outputs.fbthrift_CACHE_KEY }}-install - name: Build fbthrift if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fbthrift - name: Save fbthrift to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} @@ -613,7 +618,7 @@ jobs: key: ${{ steps.paths.outputs.fb303_CACHE_KEY }}-install - name: Build fb303 if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fb303 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fb303 - name: Save fb303 to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} @@ -629,7 +634,7 @@ jobs: key: ${{ steps.paths.outputs.rust-shed_CACHE_KEY }}-install - name: Build rust-shed if: ${{ steps.paths.outputs.rust-shed_SOURCE && ! steps.restore_rust-shed.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rust-shed + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests rust-shed - name: Save rust-shed to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.rust-shed_SOURCE && ! steps.restore_rust-shed.outputs.cache-hit }} @@ -637,7 +642,7 @@ jobs: path: ${{ steps.paths.outputs.rust-shed_INSTALL }} key: ${{ steps.paths.outputs.rust-shed_CACHE_KEY }}-install - name: Build mononoke - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. mononoke --project-install-prefix mononoke:/ + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --src-dir=. mononoke --project-install-prefix mononoke:/ - name: Copy artifacts run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. mononoke _artifacts/linux --project-install-prefix mononoke:/ --final-install-prefix /usr/local - uses: actions/upload-artifact@v4 @@ -645,7 +650,7 @@ jobs: name: mononoke path: _artifacts - name: Test mononoke - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/ + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --build-type MinSizeRel --src-dir=. mononoke --project-install-prefix mononoke:/ - name: Show disk space at end if: always() run: df -h diff --git a/.github/workflows/mononoke_mac.yml b/.github/workflows/mononoke_mac.yml index 128bf4ec7357b..9413b35ec3743 100644 --- a/.github/workflows/mononoke_mac.yml +++ b/.github/workflows/mononoke_mac.yml @@ -24,8 +24,8 @@ jobs: - id: paths name: Query paths run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. mononoke >> "$GITHUB_OUTPUT" - - name: Install Rust Stable - uses: dtolnay/rust-toolchain@stable + - name: Install Rust 1.91 + uses: dtolnay/rust-toolchain@1.91 - name: Fetch xxhash if: ${{ steps.paths.outputs.xxhash_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash @@ -125,7 +125,7 @@ jobs: key: ${{ steps.paths.outputs.xxhash_CACHE_KEY }}-install - name: Build xxhash if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests xxhash - name: Save xxhash to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} @@ -141,7 +141,7 @@ jobs: key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install - name: Build openssl if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests openssl - name: Save openssl to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} @@ -157,7 +157,7 @@ jobs: key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - name: Build ninja if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests ninja - name: Save ninja to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} @@ -173,7 +173,7 @@ jobs: key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - name: Build cmake if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests cmake - name: Save cmake to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} @@ -189,7 +189,7 @@ jobs: key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install - name: Build fmt if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fmt - name: Save fmt to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} @@ -205,7 +205,7 @@ jobs: key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install - name: Build googletest if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests googletest - name: Save googletest to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} @@ -221,7 +221,7 @@ jobs: key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install - name: Build zstd if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests zstd - name: Save zstd to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} @@ -237,7 +237,7 @@ jobs: key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install - name: Build boost if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests boost - name: Save boost to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} @@ -253,7 +253,7 @@ jobs: key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install - name: Build double-conversion if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests double-conversion - name: Save double-conversion to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} @@ -269,7 +269,7 @@ jobs: key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install - name: Build fast_float if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fast_float + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fast_float - name: Save fast_float to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} @@ -285,7 +285,7 @@ jobs: key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install - name: Build gflags if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests gflags - name: Save gflags to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} @@ -301,7 +301,7 @@ jobs: key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install - name: Build glog if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests glog - name: Save glog to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} @@ -317,7 +317,7 @@ jobs: key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install - name: Build libdwarf if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libdwarf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libdwarf - name: Save libdwarf to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} @@ -333,7 +333,7 @@ jobs: key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install - name: Build libevent if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libevent - name: Save libevent to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} @@ -349,7 +349,7 @@ jobs: key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install - name: Build lz4 if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests lz4 - name: Save lz4 to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} @@ -365,7 +365,7 @@ jobs: key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install - name: Build snappy if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests snappy - name: Save snappy to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} @@ -381,7 +381,7 @@ jobs: key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install - name: Build liboqs if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests liboqs + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests liboqs - name: Save liboqs to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} @@ -397,7 +397,7 @@ jobs: key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install - name: Build zlib if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests zlib - name: Save zlib to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} @@ -413,7 +413,7 @@ jobs: key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - name: Build autoconf if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests autoconf - name: Save autoconf to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} @@ -429,7 +429,7 @@ jobs: key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - name: Build automake if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests automake - name: Save automake to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} @@ -445,7 +445,7 @@ jobs: key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - name: Build libtool if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libtool - name: Save libtool to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} @@ -461,7 +461,7 @@ jobs: key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install - name: Build libsodium if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests libsodium - name: Save libsodium to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} @@ -477,7 +477,7 @@ jobs: key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install - name: Build xz if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests xz - name: Save xz to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} @@ -493,7 +493,7 @@ jobs: key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install - name: Build folly if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests folly - name: Save folly to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} @@ -509,7 +509,7 @@ jobs: key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install - name: Build fizz if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fizz - name: Save fizz to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} @@ -525,7 +525,7 @@ jobs: key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install - name: Build mvfst if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests mvfst - name: Save mvfst to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} @@ -541,7 +541,7 @@ jobs: key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install - name: Build wangle if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests wangle - name: Save wangle to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} @@ -557,7 +557,7 @@ jobs: key: ${{ steps.paths.outputs.fbthrift_CACHE_KEY }}-install - name: Build fbthrift if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fbthrift - name: Save fbthrift to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} @@ -573,7 +573,7 @@ jobs: key: ${{ steps.paths.outputs.fb303_CACHE_KEY }}-install - name: Build fb303 if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fb303 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests fb303 - name: Save fb303 to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} @@ -589,7 +589,7 @@ jobs: key: ${{ steps.paths.outputs.rust-shed_CACHE_KEY }}-install - name: Build rust-shed if: ${{ steps.paths.outputs.rust-shed_SOURCE && ! steps.restore_rust-shed.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rust-shed + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --free-up-disk --no-tests rust-shed - name: Save rust-shed to cache uses: actions/cache/save@v4 if: ${{ steps.paths.outputs.rust-shed_SOURCE && ! steps.restore_rust-shed.outputs.cache-hit }} @@ -597,7 +597,7 @@ jobs: path: ${{ steps.paths.outputs.rust-shed_INSTALL }} key: ${{ steps.paths.outputs.rust-shed_CACHE_KEY }}-install - name: Build mononoke - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. mononoke --project-install-prefix mononoke:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --build-type MinSizeRel --src-dir=. mononoke --project-install-prefix mononoke:/usr/local - name: Copy artifacts run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. mononoke _artifacts/mac --project-install-prefix mononoke:/usr/local --final-install-prefix /usr/local - uses: actions/upload-artifact@v4 @@ -605,7 +605,7 @@ jobs: name: mononoke path: _artifacts - name: Test mononoke - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --build-type MinSizeRel --src-dir=. mononoke --project-install-prefix mononoke:/usr/local - name: Show disk space at end if: always() run: df -h diff --git a/.github/workflows/sapling-cli-getdeps_linux.yml b/.github/workflows/sapling-cli-getdeps_linux.yml index 4665d0be1d979..e36d3fb3632dd 100644 --- a/.github/workflows/sapling-cli-getdeps_linux.yml +++ b/.github/workflows/sapling-cli-getdeps_linux.yml @@ -3,7 +3,12 @@ name: Sapling CLI Getdeps Linux on: - workflow_dispatch + push: + branches: + - main + pull_request: + branches: + - main permissions: contents: read # to fetch code (actions/checkout) diff --git a/addons/isl/src/Bookmark.tsx b/addons/isl/src/Bookmark.tsx index f85953e8cb22d..1fa66af00f951 100644 --- a/addons/isl/src/Bookmark.tsx +++ b/addons/isl/src/Bookmark.tsx @@ -28,7 +28,6 @@ import { REMOTE_MASTER_BOOKMARK, } from './BookmarksData'; import {Row} from './ComponentUtils'; -import {useFeatureFlagSync} from './featureFlags'; import {T, t} from './i18n'; import {Internal} from './Internal'; import {BookmarkCreateOperation} from './operations/BookmarkCreateOperation'; @@ -144,7 +143,6 @@ export function AllBookmarksTruncated({ }) { const bookmarksData = useAtomValue(bookmarksDataStorage); const recommendedBookmarks = useAtomValue(recommendedBookmarksAtom); - const recommendedBookmarksGK = useFeatureFlagSync(Internal.featureFlags?.RecommendedBookmarks); const recommendedBookmarksAvailable = useAtomValue(recommendedBookmarksAvailableAtom); const FullRepoBranchBookmark = Internal.FullRepoBranchBookmark; @@ -171,9 +169,8 @@ export function AllBookmarksTruncated({ .map(bookmark => { const value = typeof bookmark === 'string' ? bookmark : bookmark.value; const isRecommended = - recommendedBookmarksGK && - (recommendedBookmarks.has(value) || - (typeof bookmark === 'object' && bookmark.isRecommended === true)); + recommendedBookmarks.has(value) || + (typeof bookmark === 'object' && bookmark.isRecommended === true); const tooltipOverride = typeof bookmark === 'string' ? undefined : bookmark.description; const {icon, tooltip} = getBookmarkAddons( value, diff --git a/addons/isl/src/BookmarksData.tsx b/addons/isl/src/BookmarksData.tsx index f768114848cc5..4f8778e1e69e5 100644 --- a/addons/isl/src/BookmarksData.tsx +++ b/addons/isl/src/BookmarksData.tsx @@ -10,7 +10,6 @@ import type {StableLocationData} from './types'; import {atom} from 'jotai'; import {tracker} from './analytics'; import serverAPI from './ClientToServerAPI'; -import {featureFlagLoadable} from './featureFlags'; import {Internal} from './Internal'; import {lazyAtom, localStorageBackedAtom, readAtom, writeAtom} from './jotaiUtils'; import {latestCommits} from './serverAPIState'; @@ -124,11 +123,6 @@ export const recommendedBookmarksOnboarding = localStorageBackedAtom( true, ); -export const recommendedBookmarksGKAtom = atom(get => { - const flag = get(featureFlagLoadable(Internal.featureFlags?.RecommendedBookmarks)); - return flag.state === 'hasData' ? flag.data : false; -}); - export const recommendedBookmarksAtom = lazyAtom(async _get => { const recommendedBookmarks = await (Internal.getRecommendedBookmarks?.() ?? Promise.resolve(new Set())); diff --git a/addons/isl/src/BookmarksManager.tsx b/addons/isl/src/BookmarksManager.tsx index 9b90041f89de7..cd01529472c0e 100644 --- a/addons/isl/src/BookmarksManager.tsx +++ b/addons/isl/src/BookmarksManager.tsx @@ -42,7 +42,6 @@ import {Column, Row, ScrollY} from './ComponentUtils'; import {DropdownFields} from './DropdownFields'; import {useCommandEvent} from './ISLShortcuts'; import {Internal} from './Internal'; -import {useFeatureFlagSync} from './featureFlags'; import {T, t} from './i18n'; import {readAtom} from './jotaiUtils'; import {latestDag} from './serverAPIState'; @@ -100,10 +99,7 @@ function BookmarksManager(_props: {dismiss: () => void}) { const bookmarksData = useAtomValue(bookmarksDataStorage); const recommendedBookmarks = useAtomValue(recommendedBookmarksAtom); const recommendedBookmarksAvailable = useAtomValue(recommendedBookmarksAvailableAtom); - const enableRecommended = - useFeatureFlagSync(Internal.featureFlags?.RecommendedBookmarks) && - bookmarksData.useRecommendedBookmark && - recommendedBookmarksAvailable; + const enableRecommended = bookmarksData.useRecommendedBookmark && recommendedBookmarksAvailable; // Place recommended bookmarks (and remote/master) first if enabled, then the rest const priority = new Set(recommendedBookmarks).add(REMOTE_MASTER_BOOKMARK); @@ -347,7 +343,6 @@ function BookmarksList({ kind: BookmarkKind; }) { const [bookmarksData, setBookmarksData] = useAtom(bookmarksDataStorage); - const recommendedBookmarksGK = useFeatureFlagSync(Internal.featureFlags?.RecommendedBookmarks); const recommendedBookmarks = useAtomValue(recommendedBookmarksAtom); const recommendedBookmarksAvailable = useAtomValue(recommendedBookmarksAvailableAtom); @@ -364,10 +359,8 @@ function BookmarksList({ const name = typeof bookmark === 'string' ? bookmark : bookmark.name; const extra = typeof bookmark === 'string' ? undefined : bookmark.extra; const enableRecommended = - recommendedBookmarksGK && - bookmarksData.useRecommendedBookmark && - recommendedBookmarksAvailable; - const isRecommended = recommendedBookmarksGK && recommendedBookmarks.has(name); + bookmarksData.useRecommendedBookmark && recommendedBookmarksAvailable; + const isRecommended = recommendedBookmarks.has(name); const tooltipOverride = typeof bookmark === 'string' ? undefined : bookmark.info; const {icon, tooltip} = getBookmarkAddons( name, diff --git a/addons/isl/src/UncommittedChanges.tsx b/addons/isl/src/UncommittedChanges.tsx index 010ea2ff35471..9a89e66634d1a 100644 --- a/addons/isl/src/UncommittedChanges.tsx +++ b/addons/isl/src/UncommittedChanges.tsx @@ -877,7 +877,13 @@ function MergeConflictButtons({ key="continue" disabled={!allConflictsResolved || shouldDisableButtons} data-testid="conflict-continue-button" - onClick={() => { + onClick={async () => { + const conflictFiles = + conflicts.state === 'loaded' ? conflicts.files.map(f => f.path) : []; + if (!(await confirmSuggestedEditsForFiles('merge-continue', 'accept', conflictFiles))) { + return; + } + if (readAtom(shouldAutoResolveAllBeforeContinue)) { runOperation(new RunMergeDriversOperation()); } diff --git a/addons/isl/src/serverAPIState.ts b/addons/isl/src/serverAPIState.ts index d0ced2fa098a2..eee8e8c2deb49 100644 --- a/addons/isl/src/serverAPIState.ts +++ b/addons/isl/src/serverAPIState.ts @@ -32,7 +32,6 @@ import { bookmarksDataStorage, recommendedBookmarksAtom, recommendedBookmarksAvailableAtom, - recommendedBookmarksGKAtom, REMOTE_MASTER_BOOKMARK, } from './BookmarksData'; import serverAPI from './ClientToServerAPI'; @@ -307,10 +306,7 @@ export const latestDag = atom(get => { const successorMap = get(latestSuccessorsMapAtom); const bookmarksData = get(bookmarksDataStorage); const recommendedBookmarksAvailable = get(recommendedBookmarksAvailableAtom); - const enableRecommended = - get(recommendedBookmarksGKAtom) && - bookmarksData.useRecommendedBookmark && - recommendedBookmarksAvailable; + const enableRecommended = bookmarksData.useRecommendedBookmark && recommendedBookmarksAvailable; const recommendedBookmarks = get(recommendedBookmarksAtom); const commitDag = undefined; // will be populated from `commits` diff --git a/addons/isl/src/sloc/useFetchSignificantLinesOfCode.ts b/addons/isl/src/sloc/useFetchSignificantLinesOfCode.ts index 07bbc82445ef4..bd2c628433f29 100644 --- a/addons/isl/src/sloc/useFetchSignificantLinesOfCode.ts +++ b/addons/isl/src/sloc/useFetchSignificantLinesOfCode.ts @@ -11,7 +11,7 @@ import type {CommitInfo, RepoRelativePath, SlocInfo} from '../types'; import {atom, useAtomValue} from 'jotai'; import {loadable} from 'jotai/utils'; -import {useRef} from 'react'; +import {useEffect, useMemo, useRef, useState} from 'react'; import serverAPI from '../ClientToServerAPI'; import {commitInfoViewCurrentCommits} from '../CommitInfoView/CommitInfoState'; import {getGeneratedFilesFrom} from '../GeneratedFile'; @@ -168,12 +168,6 @@ const fetchPendingAmendSloc = async ( return slocInfo; }; -let pendingAmendRequestId = 0; -const pendingAmendSlocAtom = atom(get => { - const selectedFiles = get(selectedFilesAtom); - return fetchPendingAmendSloc(get, selectedFiles, pendingAmendRequestId++); -}); - /** * FETCH PENDING SLOC */ @@ -230,16 +224,6 @@ const fetchPendingSloc = async ( return pendingLocData; }; -let pendingRequestId = 0; - -const pendingChangesSlocAtom = atom(get => { - const selectedFiles = get(selectedFilesAtom); - return fetchPendingSloc(get, selectedFiles, pendingRequestId++); -}); - -const pendingAmendSlocLoadableAtom = loadable(pendingAmendSlocAtom); -const pendingChangesSlocLoadableAtom = loadable(pendingChangesSlocAtom); - function useFetchWithPrevious(atom: Atom>>): { slocInfo: SlocInfo | undefined; isLoading: boolean; @@ -274,10 +258,65 @@ export function useFetchSignificantLinesOfCode(commit: CommitInfo) { return {slocInfo: result.data, isLoading: false}; } +// Debounce delay for SLOC requests to prevent spamming when many files are selected quickly +const DEBOUNCE_DELAY_MS = 300; + +// Hook that debounces an atom value +function useDebouncedAtomValue(sourceAtom: Atom, debounceMs: number): T { + const currentValue = useAtomValue(sourceAtom); + const [debouncedValue, setDebouncedValue] = useState(currentValue); + const timeoutRef = useRef | null>(null); + + useEffect(() => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + } + + timeoutRef.current = setTimeout(() => { + setDebouncedValue(currentValue); + }, debounceMs); + + return () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + } + }; + }, [currentValue, debounceMs]); + + return debouncedValue; +} + +let pendingRequestId = 0; export function useFetchPendingSignificantLinesOfCode() { - return useFetchWithPrevious(pendingChangesSlocLoadableAtom); + // Debounce selected files to prevent spamming SLOC requests + const debouncedSelectedFiles = useDebouncedAtomValue(selectedFilesAtom, DEBOUNCE_DELAY_MS); + + // Use a derived atom that depends on the debounced value + const debouncedAtom = useMemo(() => { + return atom(get => { + // Force the atom to use the debounced value by creating a dependency + // Note: we can't pass debouncedSelectedFiles directly to the atom, + // so we create a new fetch call with it + return fetchPendingSloc(get, debouncedSelectedFiles, pendingRequestId++); + }); + }, [debouncedSelectedFiles]); + + const loadableAtom = loadable(debouncedAtom); + return useFetchWithPrevious(loadableAtom); } +let pendingAmendRequestId = 0; export function useFetchPendingAmendSignificantLinesOfCode() { - return useFetchWithPrevious(pendingAmendSlocLoadableAtom); + // Debounce selected files to prevent spamming SLOC requests + const debouncedSelectedFiles = useDebouncedAtomValue(selectedFilesAtom, DEBOUNCE_DELAY_MS); + + // Use a derived atom that depends on the debounced value + const debouncedAtom = useMemo(() => { + return atom(get => { + return fetchPendingAmendSloc(get, debouncedSelectedFiles, pendingAmendRequestId++); + }); + }, [debouncedSelectedFiles]); + + const loadableAtom = loadable(debouncedAtom); + return useFetchWithPrevious(loadableAtom); } diff --git a/build/deps/github_hashes/facebook/fb303-rev.txt b/build/deps/github_hashes/facebook/fb303-rev.txt index 3de32deea6a90..b57afd36e5ce7 100644 --- a/build/deps/github_hashes/facebook/fb303-rev.txt +++ b/build/deps/github_hashes/facebook/fb303-rev.txt @@ -1 +1 @@ -Subproject commit faed614f7626964abc445ff94e467a8d5de90b10 +Subproject commit 1aca35e24e638a0db4c9e70817af432634e16f5e diff --git a/build/deps/github_hashes/facebook/fbthrift-rev.txt b/build/deps/github_hashes/facebook/fbthrift-rev.txt index 1bba2b162a71e..18e68c6175168 100644 --- a/build/deps/github_hashes/facebook/fbthrift-rev.txt +++ b/build/deps/github_hashes/facebook/fbthrift-rev.txt @@ -1 +1 @@ -Subproject commit 4ce92f46fc271e173fce705a3b790ee14149751c +Subproject commit 674867a3c8e254f5be9519d26a7a5b531932c62a diff --git a/build/deps/github_hashes/facebook/folly-rev.txt b/build/deps/github_hashes/facebook/folly-rev.txt index 7c9f4dcf7f688..b234617932218 100644 --- a/build/deps/github_hashes/facebook/folly-rev.txt +++ b/build/deps/github_hashes/facebook/folly-rev.txt @@ -1 +1 @@ -Subproject commit 3f1fa35e63aee37dd25857d13d796857186b3aa0 +Subproject commit 6a4ddcd5ccd0bc91b25d3f3c1820d556ff975cf1 diff --git a/build/deps/github_hashes/facebook/mvfst-rev.txt b/build/deps/github_hashes/facebook/mvfst-rev.txt new file mode 100644 index 0000000000000..78b62ff1c60c0 --- /dev/null +++ b/build/deps/github_hashes/facebook/mvfst-rev.txt @@ -0,0 +1 @@ +Subproject commit a5683c792479a16d2faae915fffd6a2b29559d37 diff --git a/build/deps/github_hashes/facebook/wangle-rev.txt b/build/deps/github_hashes/facebook/wangle-rev.txt index 762f64506fe99..c00c0e4011b1a 100644 --- a/build/deps/github_hashes/facebook/wangle-rev.txt +++ b/build/deps/github_hashes/facebook/wangle-rev.txt @@ -1 +1 @@ -Subproject commit 34f740cfa86fd27fa039487071d319b46cab1ca2 +Subproject commit 45ff542aca0d2449bc5d3e150d2aae297e761d6d diff --git a/build/deps/github_hashes/facebookexperimental/edencommon-rev.txt b/build/deps/github_hashes/facebookexperimental/edencommon-rev.txt index b285ab55954a9..3b42f457e1ec8 100644 --- a/build/deps/github_hashes/facebookexperimental/edencommon-rev.txt +++ b/build/deps/github_hashes/facebookexperimental/edencommon-rev.txt @@ -1 +1 @@ -Subproject commit 414b40088d33efbc70b0b19591a8e82771462cd0 +Subproject commit 3fd57cafad19ed5fe6f3dd534868b92bb804e1dc diff --git a/build/deps/github_hashes/facebookexperimental/rust-shed-rev.txt b/build/deps/github_hashes/facebookexperimental/rust-shed-rev.txt index 119226090ffa1..1a5bd375fa2a7 100644 --- a/build/deps/github_hashes/facebookexperimental/rust-shed-rev.txt +++ b/build/deps/github_hashes/facebookexperimental/rust-shed-rev.txt @@ -1 +1 @@ -Subproject commit 4f3216b7bbfa27da8a3920b1c28f851ca8a5e1b6 +Subproject commit cac1924e50c59baf47428538f99288bbc82bcd0e diff --git a/build/deps/github_hashes/facebookincubator/fizz-rev.txt b/build/deps/github_hashes/facebookincubator/fizz-rev.txt index 8994a461e2d2a..eb6f41f31313f 100644 --- a/build/deps/github_hashes/facebookincubator/fizz-rev.txt +++ b/build/deps/github_hashes/facebookincubator/fizz-rev.txt @@ -1 +1 @@ -Subproject commit 05415a3e812e3116d08006701381e632c2cbf206 +Subproject commit 01b31be94b35e1e000baa2b40942403e78e4b935 diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 4c9a66460ae06..b99b1c6ace1cf 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -964,6 +964,13 @@ def setup_project_cmd_parser(self, parser): default=None, help="Timeout in seconds for each individual test", ) + parser.add_argument( + "--build-type", + help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", + choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], + action="store", + default=None, + ) @cmd( @@ -1049,6 +1056,13 @@ def write_job_for_platform(self, platform, args): # noqa: C901 args.enable_tests and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off" ) + rust_version = ( + manifest.get("github.actions", "rust_version", ctx=manifest_ctx) or "stable" + ) + + override_build_type = args.build_type or manifest.get( + "github.actions", "build_type", ctx=manifest_ctx + ) if run_tests: manifest_ctx.set("test", "on") run_on = self.get_run_on(args) @@ -1163,8 +1177,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(" - uses: actions/checkout@v4\n") build_type_arg = "" - if args.build_type: - build_type_arg = f"--build-type {args.build_type} " + if override_build_type: + build_type_arg = f"--build-type {override_build_type} " if build_opts.free_up_disk: free_up_disk = "--free-up-disk " @@ -1241,8 +1255,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 or builder_name == "cargo" or mbuilder_name == "cargo" ): - out.write(" - name: Install Rust Stable\n") - out.write(" uses: dtolnay/rust-toolchain@stable\n") + out.write(f" - name: Install Rust {rust_version.capitalize()}\n") + out.write(f" uses: dtolnay/rust-toolchain@{rust_version}\n") break # Normal deps that have manifests @@ -1360,7 +1374,7 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(" - name: Test %s\n" % manifest.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} test {num_jobs_arg}--src-dir=. {manifest.name}{project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} test {build_type_arg}{num_jobs_arg}--src-dir=. {manifest.name}{project_prefix}\n" ) if build_opts.free_up_disk and not build_opts.is_windows(): out.write(" - name: Show disk space at end\n") diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 561875b65e70b..20ae55fe42757 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -95,7 +95,7 @@ def cargo_config_file(self): if self.cargo_config_file_subdir: return os.path.join(build_source_dir, self.cargo_config_file_subdir) else: - return os.path.join(build_source_dir, ".cargo", "config") + return os.path.join(build_source_dir, ".cargo", "config.toml") def _create_cargo_config(self): cargo_config_file = self.cargo_config_file() @@ -122,8 +122,12 @@ def _create_cargo_config(self): [profile.dev] debug = false incremental = false + +[profile.release] +opt-level = "{}" """.format( - self.build_dir.replace("\\", "\\\\") + self.build_dir.replace("\\", "\\\\"), + "z" if self.build_opts.build_type == "MinSizeRel" else "s", ) # Point to vendored sources from getdeps manifests @@ -173,7 +177,7 @@ def _build(self, reconfigure) -> None: build_source_dir = self.build_source_dir() build_args = [ - "--out-dir", + "--artifact-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options", ] @@ -206,20 +210,26 @@ def _build(self, reconfigure) -> None: def run_tests( self, schedule_type, owner, test_filter, retry, no_testpilot, timeout=None ) -> None: + build_args = [] + if self.build_opts.build_type != "Debug": + build_args.append("--release") + if test_filter: - args = ["--", test_filter] + filter_args = ["--", test_filter] else: - args = [] + filter_args = [] if self.manifests_to_build is None: - self.run_cargo(self.install_dirs, "test", args) - if self.build_doc: + self.run_cargo(self.install_dirs, "test", build_args + filter_args) + if self.build_doc and not filter_args: self.run_cargo(self.install_dirs, "doc", ["--no-deps"]) else: for manifest in self.manifests_to_build: margs = ["--manifest-path", self.manifest_dir(manifest)] - self.run_cargo(self.install_dirs, "test", args + margs) - if self.build_doc: + self.run_cargo( + self.install_dirs, "test", build_args + filter_args + margs + ) + if self.build_doc and not filter_args: self.run_cargo(self.install_dirs, "doc", ["--no-deps"] + margs) def _patchup_workspace(self, dep_to_git) -> None: diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 3441aac3faf22..99479e68c9e72 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -98,6 +98,8 @@ "fields": { "run_tests": OPTIONAL, "required_locales": OPTIONAL, + "rust_version": OPTIONAL, + "build_type": OPTIONAL, }, }, "crate.pathmap": {"optional_section": True}, diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke index d9c88454dfa7f..de1c4cdf5dca6 100644 --- a/build/fbcode_builder/manifests/mononoke +++ b/build/fbcode_builder/manifests/mononoke @@ -18,6 +18,10 @@ builder = nop build_doc = true workspace_dir = eden/mononoke +[github.actions] +rust_version = 1.91 +build_type = MinSizeRel + [shipit.pathmap] fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg diff --git a/configerator/structs/scm/mononoke/ratelimiting/ratelimits.thrift b/configerator/structs/scm/mononoke/ratelimiting/ratelimits.thrift index 73ba1116b46f4..ce3a8c631497d 100644 --- a/configerator/structs/scm/mononoke/ratelimiting/ratelimits.thrift +++ b/configerator/structs/scm/mononoke/ratelimiting/ratelimits.thrift @@ -1,4 +1,4 @@ -// @generated SignedSource<> +// @generated SignedSource<<5fb9324c164a21d78e98fbe5335fa8c5>> // DO NOT EDIT THIS FILE MANUALLY! // This file is a mechanical copy of the version in the configerator repo. To // modify it, edit the copy in the configerator repo instead and copy it over by @@ -30,8 +30,15 @@ union Target { // A list of client's identities, such as ["MACHINE_TIER:sandcastle"] // All must be present for the rule to apply 7: list identities; + // Atlas environment target, for rate limiting Atlas workloads + 8: AtlasTarget atlas; } +// Target for Atlas environments. Applies to all Atlas environments. +// Rate limits will still be applied per-main-client-id basis. +@rust.Exhaustive +struct AtlasTarget {} + // We do not want static slices to use other StaticSlices as a target. // At the same time I want to avoid verbosity when defining Targets union StaticSliceTarget { diff --git a/eden/fs/BUCK b/eden/fs/BUCK index 939bf42612a9c..855b065b18715 100644 --- a/eden/fs/BUCK +++ b/eden/fs/BUCK @@ -1,3 +1,5 @@ +load("@fbcode//cpe/nupkg_builder:nupkg.bzl", "nupkg") +load("@fbcode//eden/fs:build_targets.bzl", "make_rpm_features") load("@fbcode//registry:defs.bzl", "rpm") load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") @@ -21,7 +23,6 @@ rpm.builder( autoclean_keep_versions = 200, buck_opts = rpm.buck_opts( config = { - "fbcode.dwp": "true", # split-dwarf is required for reduce binary size "fbcode.split-dwarf": "true", }, @@ -41,96 +42,32 @@ rpm.builder( repo = "darwin/any", ), ], - features = [ - rpm.install( - src = "fbcode//eden/fs/service:edenfs", - dst = "/usr/local/libexec/eden/edenfs", - ), - rpm.install( - src = "fbcode//eden/fs/service:edenfs_privhelper", - dst = "/usr/local/libexec/eden/edenfs_privhelper", - mode = 0o04755, - ), - rpm.install( - src = "fbcode//eden/fs/store:eden_store_util", - dst = "/usr/local/libexec/eden/eden_store_util", - ), - rpm.install( - src = "fbcode//eden/fs/cli:edenfsctl", - dst = "/usr/local/bin/edenfsctl.real", - ), - rpm.install( - src = "fbcode//eden/fs/cli_rs/edenfsctl:edenfsctl", - dst = "/usr/local/bin/edenfsctl", - ), - rpm.install( - src = "fbcode//eden/fs/facebook:eden-fb303-collector", - dst = "/usr/local/libexec/eden/eden-fb303-collector", - ), - rpm.install( - src = "fbcode//eden/fs/facebook:edenfs_restarter", - dst = "/usr/local/libexec/eden/edenfs_restarter", - ), - rpm.install( - src = "fbcode//eden/fs/config/facebook:edenfs_config_manager", - dst = "/usr/local/libexec/eden/edenfs_config_manager", - ), - rpm.install( - src = "fbcode//eden/fs/monitor:edenfs_monitor", - dst = "/usr/local/libexec/eden/edenfs_monitor", - ), - rpm.install( - src = "fbcode//eden/fs/cli/trace:trace_stream", - dst = "/usr/local/libexec/eden/eden_trace_stream", - ), - rpm.install( - src = "fbcode//eden/fs/inodes/fscatalog:eden_fsck", - dst = "/usr/local/libexec/eden/eden_fsck", - ), - rpm.install( - src = "facebook/packaging/config.d/00-defaults.toml", - dst = "/etc/eden/config.d/00-defaults.toml", - mode = 0o0755, - ), - rpm.install( - src = "facebook/packaging/config.d/doctor.toml", - dst = "/etc/eden/config.d/doctor.toml", - mode = 0o0755, - ), - rpm.install( - src = "facebook/packaging/ignore", - dst = "/etc/eden/ignore", - mode = 0o0755, - ), - rpm.install( - src = "facebook/packaging/NOT_MOUNTED_README.txt", - dst = "/etc/eden/NOT_MOUNTED_README.txt", - ), - rpm.install( - src = "scripts/facebook/rg_perf_test", - dst = "/usr/local/libexec/eden/eden_rg_perf_script", - mode = 0o0755, - ), - rpm.install( - src = "scripts/facebook/eden_bench.sh", - dst = "/usr/local/libexec/eden/eden_bench.sh", - mode = 0o0755, - ), - rpm.install( - src = "scripts/facebook/eden_prof", - dst = "/usr/local/libexec/eden/eden_prof", - mode = 0o0755, - ), - rpm.file_symlink( - link = "/usr/local/bin/eden", - target = "/usr/local/bin/edenfsctl", - ), - rpm.ensure_dirs_exist( - dirs = "/etc/eden", - ), - rpm.ensure_dirs_exist( - dirs = "/etc/eden/config.d", - ), + features = make_rpm_features(), + summary = "Source control aware virtual filesystem", +) + +nupkg.builder( + name = "fb-eden-test-windows", + compatible_with = [ + "ovr_config//os:windows", + ], + oncall = "scm_client_infra", + shims = [ + { + nupkg.shim.source: "fbcode//eden/fs/cli_rs/edenfsctl:edenfsctl", + nupkg.shim.name: "eden", + }, ], summary = "Source control aware virtual filesystem", + deps = [ + "fbcode//eden/fs/cli:edenfsctl", + "fbcode//eden/fs/cli/trace:trace_stream", + "fbcode//eden/fs/cli_rs/edenfsctl:edenfsctl", + "fbcode//eden/fs/config/facebook:edenfs_config_manager", + "fbcode//eden/fs/facebook:eden-fb303-collector", + "fbcode//eden/fs/service:edenfs", + "fbcode//eden/fs/service:edenfs[pdb]", + # TODO: Figure out symbol package "//arvr/tools/translator:symbol_ents" + # TODO: Set up install script to copy files to the appropriate locations + ], ) diff --git a/eden/fs/build_targets.bzl b/eden/fs/build_targets.bzl new file mode 100644 index 0000000000000..b0a88eb977fa4 --- /dev/null +++ b/eden/fs/build_targets.bzl @@ -0,0 +1,98 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2. + +# Build targets for EdenFS packaging. +# Maps build targets to their installation paths from packman.yml + +# Main EdenFS build targets used in packaging +# Maps build target -> install path(s) + +load("@fbcode//registry:defs.bzl", "rpm") + +EDENFS_TARGETS = { + "//eden/fs/cli/trace:trace_stream": "/usr/local/libexec/eden/eden_trace_stream", + "//eden/fs/cli:edenfsctl": "/usr/local/bin/edenfsctl.real", + "//eden/fs/cli_rs/edenfsctl:edenfsctl": "/usr/local/bin/edenfsctl", + "//eden/fs/config/facebook:edenfs_config_manager": "/usr/local/libexec/eden/edenfs_config_manager", + "//eden/fs/facebook:eden-fb303-collector": "/usr/local/libexec/eden/eden-fb303-collector", + "//eden/fs/facebook:edenfs_restarter": "/usr/local/libexec/eden/edenfs_restarter", + "//eden/fs/inodes/fscatalog:eden_fsck": "/usr/local/libexec/eden/eden_fsck", + "//eden/fs/monitor:edenfs_monitor": "/usr/local/libexec/eden/edenfs_monitor", + "//eden/fs/service:edenfs": "/usr/local/libexec/eden/edenfs", + "//eden/fs/service:edenfs_privhelper": "/usr/local/libexec/eden/edenfs_privhelper", + "//eden/fs/store:eden_store_util": "/usr/local/libexec/eden/eden_store_util", +} + +SYMLINKS = { + "/usr/local/bin/edenfsctl": "/usr/local/bin/eden", +} + +STATIC_TARGETS = { + "facebook/packaging/NOT_MOUNTED_README.txt": "/etc/eden/NOT_MOUNTED_README.txt", + "facebook/packaging/ignore": "/etc/eden/ignore", +} + +SCRIPTS_TARGETS = { + "scripts/facebook/eden_bench.sh": "/usr/local/libexec/eden/eden_bench.sh", + "scripts/facebook/eden_prof": "/usr/local/libexec/eden/eden_prof", + "scripts/facebook/rg_perf_test": "/usr/local/libexec/eden/eden_rg_perf_script", +} + +CONFIG_D_TARGETS = { + "facebook/packaging/config.d/00-defaults.toml": "/etc/eden/config.d/00-defaults.toml", + "facebook/packaging/config.d/doctor.toml": "/etc/eden/config.d/doctor.toml", +} + +TARGET_MODES = { + "//eden/fs/service:edenfs_privhelper": 0o04755, + "//eden/scm/exec/eden_apfs_mount_helper:eden_apfs_mount_helper": 0o04755, + "facebook/packaging/ignore": 0o0755, +} + +DIRS = [ + "/etc/eden", + "/etc/eden/config.d", +] + +MAC_TARGETS = { + "//eden/scm/exec/eden_apfs_mount_helper:eden_apfs_mount_helper": "/usr/local/libexec/eden/eden_apfs_mount_helper", +} + +def make_rpm_features(): + features = [] + for target, install_path in EDENFS_TARGETS.items(): + if target in TARGET_MODES: + features.append(rpm.install(src = "fbcode" + target, dst = install_path, mode = TARGET_MODES.get(target))) + else: + features.append(rpm.install(src = "fbcode" + target, dst = install_path)) + if install_path in SYMLINKS: + features.append(rpm.file_symlink(link = SYMLINKS.get(install_path), target = install_path)) + for dir in DIRS: + features.append(rpm.ensure_dirs_exist(dir)) + for target, install_path in STATIC_TARGETS.items(): + if target in TARGET_MODES: + features.append(rpm.install(src = target, dst = install_path, mode = TARGET_MODES.get(target))) + else: + features.append(rpm.install(src = target, dst = install_path)) + for target, install_path in SCRIPTS_TARGETS.items(): + features.append(rpm.install(src = target, dst = install_path, mode = 0o0755)) + for target, install_path in CONFIG_D_TARGETS.items(): + features.append(rpm.install(src = target, dst = install_path, mode = 0o0755)) + + mac_features = [] + + for target, install_path in MAC_TARGETS.items(): + if target in TARGET_MODES: + mac_features.append(rpm.install(src = "fbcode" + target, dst = install_path, mode = TARGET_MODES.get(target))) + else: + mac_features.append(rpm.install(src = "fbcode" + target, dst = install_path)) + for mac_feature in mac_features: + features.append( + select({ + "DEFAULT": None, + "ovr_config//os:macos": mac_feature, + }), + ) + return features diff --git a/eden/fs/cli/edenfsctl_strategy+full.marshal b/eden/fs/cli/edenfsctl_strategy+full.marshal index 439b81dcdd571..db5be444bcf0d 100644 Binary files a/eden/fs/cli/edenfsctl_strategy+full.marshal and b/eden/fs/cli/edenfsctl_strategy+full.marshal differ diff --git a/eden/mononoke/common/scuba_ext/src/lib.rs b/eden/mononoke/common/scuba_ext/src/lib.rs index b80606747916f..a34cb897a2154 100644 --- a/eden/mononoke/common/scuba_ext/src/lib.rs +++ b/eden/mononoke/common/scuba_ext/src/lib.rs @@ -175,7 +175,7 @@ impl MononokeScubaSampleBuilder { }, ExperimentJKData { jk_name: "scm/mononoke:remote_diff", - switch_values: vec!["instagram-server"], + switch_values: vec!["instagram-server", "www", "fbcode"], consistent_hashing: Some(client_info.correlator.as_str()), }, ExperimentJKData { diff --git a/eden/mononoke/docs/1.1-what-is-mononoke.md b/eden/mononoke/docs/1.1-what-is-mononoke.md new file mode 100644 index 0000000000000..cde071cf93700 --- /dev/null +++ b/eden/mononoke/docs/1.1-what-is-mononoke.md @@ -0,0 +1,133 @@ +# What is Mononoke? + +## Introduction + +Mononoke is a scalable source control server designed to handle massive repositories with millions of files, thousands of commits per day, and hundreds of thousands of branches. It serves as the server-side backend for Meta's source control infrastructure. + +As a core component of the Sapling source control ecosystem, Mononoke works together with EdenFS (a virtual filesystem) and the Sapling CLI to provide source control operations at large scale. + +## The Sapling Ecosystem + +Mononoke is one piece of a source control solution: + +- **Sapling CLI** - The command-line interface that developers use to interact with source control. Originally based on Mercurial, Sapling provides version control operations. + +- **EdenFS** - A virtual filesystem for massive repositories. Instead of materializing every file in a repository, EdenFS fetches files on-demand from Mononoke as they're accessed. + +- **Mononoke Server** - The server-side component that stores repository history, handles commits, manages branches (called bookmarks), and serves data to clients. + +Together, these components allow developers to work with repositories containing millions of files. + +## Why Mononoke Exists + +Traditional version control systems like Git and Mercurial were designed for smaller repositories. While they work well for projects with thousands or tens of thousands of files, they face challenges when repositories grow to millions of files and experience thousands of commits daily. + +Mononoke addresses these scalability challenges: + +**Scale in Multiple Dimensions** +- Handles millions of files +- Supports thousands of commits per hour +- Manages hundreds of thousands of active branches +- Stores files ranging from small configuration files to multi-gigabyte binaries + +**High Availability** +- Designed as a distributed service +- Uses redundant storage backends +- Can continue operating when individual components fail + +**Multi-VCS Support** +- Supports Sapling clients +- Serves Git clients using the standard Git protocol +- Maintains a single source of truth across different version control systems + +**Architectural Characteristics** +- Separates write operations from read operations +- Computes expensive indexes asynchronously +- Uses multi-level caching +- Designed for horizontal scaling + +## What Mononoke Does + +At its core, Mononoke is a source control server. When you push a commit using Sapling or Git, or when EdenFS needs to fetch file contents, they communicate with Mononoke. Mononoke handles the following operations: + +**Repository Operations** +- Accepting new commits from developers +- Serving file contents and commit history to clients +- Managing branches (bookmarks) and tags +- Enforcing pre-commit hooks and policies + +**Data Storage and Retrieval** +- Storing commit metadata and file contents in a distributed blobstore +- Maintaining mappings between different version control formats +- Managing repository state like branches and phases +- Computing and caching derived data like manifests and file history + +**Advanced Features** +- **Pushrebase** - Server-side rebasing that maintains linear history without requiring developers to manually rebase +- **Cross-repository sync** - Automatically synchronizing commits between different repositories +- **Commit Cloud** - Sharing uncommitted work across different machines +- **Hooks** - Running automated checks and validations on every commit + +## Deployment Scenarios + +Mononoke is deployed as a distributed service: + +**Server Components** +- Multiple stateless server instances handle client requests via HTTP +- Microservices handle specialized operations like landing commits or computing derived data +- Background jobs perform maintenance tasks like validation and healing + +**Storage Backends** +- Immutable blob storage (using systems like Manifold, S3, or SQL databases) for commits and files +- SQL databases for mutable state like bookmarks and mappings +- Multi-level caching (in-memory and shared caches) for performance + +**Client Types** +- Sapling CLI users pushing and pulling commits +- EdenFS instances fetching files on-demand +- Git clients using standard Git protocols +- Programmatic clients accessing data via Thrift APIs + +This distributed architecture allows Mononoke to scale horizontally—adding more servers increases capacity without architectural changes. + +## A Brief History + +Mononoke's evolution reflects changing needs in source control: + +**The Mercurial Era** +Mononoke was originally designed as a scalable backend for Mercurial, which served as Meta's primary version control system. Early Mononoke focused on supporting Mercurial clients and the Mercurial data model while introducing the Bonsai format. + +**The Sapling Transition** +As Mercurial evolved into Sapling, Mononoke evolved alongside it. The server added support for EdenAPI (now called SLAPI), an HTTP-based protocol that replaced legacy wire protocols. + +**Multi-VCS Support** +Today, Mononoke is VCS-agnostic. Rather than being tied to a specific version control system's data model, Mononoke uses Bonsai—a canonical, VCS-independent format for representing commits. This allows Mononoke to serve Git clients as well as Sapling clients, converting between formats as needed while maintaining a single source of truth. + +**Written in Rust** +Mononoke is implemented primarily in Rust, providing memory safety and concurrency support. + +## Design Principles + +Several principles guide Mononoke's design: + +**Immutability** +Most data in Mononoke is immutable and content-addressed. Once a commit or file is stored, it never changes. This allows caching, concurrent access, and data distribution across multiple storage backends. + +**Separation of Concerns** +Mononoke distinguishes between data that must be stored (commits and files) and data that can be computed (indexes and caches). This separation allows the write path to avoid computing expensive indexes synchronously, while read operations use pre-computed derived data. + +**Composability** +Rather than a monolithic repository object, Mononoke uses facets—modular components that can be composed together. Bookmark support is provided by the bookmarks facet, while commit graph traversal is handled by the commit graph facet. + +**Horizontal Scaling** +Components are designed to scale horizontally. Capacity can be increased by adding more server instances. Storage capacity can be expanded by adding more blobstore backends. + +## What's Next? + +This document provides a high-level introduction to what Mononoke is and why it exists. To dive deeper: + +- **[Key Concepts](1.2-key-concepts.md)** - Understand Bonsai, facets, derived data, and other core ideas +- **[Architecture Overview](1.3-architecture-overview.md)** - Learn how Mononoke's components fit together +- **[Navigating the Codebase](1.4-navigating-the-codebase.md)** - Find your way around Mononoke's ~55-60 directories + +This documentation is intended for developers working on Mononoke or seeking to understand how it fits into the Sapling ecosystem. diff --git a/eden/mononoke/docs/1.2-key-concepts.md b/eden/mononoke/docs/1.2-key-concepts.md new file mode 100644 index 0000000000000..71a75157d0b0a --- /dev/null +++ b/eden/mononoke/docs/1.2-key-concepts.md @@ -0,0 +1,55 @@ +# Key Concepts + +This document introduces the essential concepts you'll encounter when working with Mononoke. Each concept is covered briefly here and explored in greater depth in later sections of the documentation. + +## Bonsai + +Bonsai is Mononoke's canonical, VCS-agnostic representation of version control data. Every commit in Mononoke, regardless of whether it originated from Git, Mercurial, or Sapling, is stored internally as a Bonsai changeset. This unified format enables Mononoke to serve multiple client types from a single backend. + +A Bonsai changeset contains commit metadata (author, message, timestamps), parent changeset references, and a list of file changes (additions, modifications, deletions). Unlike Git or Mercurial, Bonsai uses a flat list of changed files rather than nested tree structures, which simplifies many operations. Each changeset is identified by a Blake2b hash of its contents, forming a Merkle DAG that ensures data integrity. File contents are stored separately as content-addressed blobs, allowing efficient deduplication across the repository. The Bonsai format was designed specifically for high-throughput writes and scalability, making it the foundation of Mononoke's performance characteristics. + +## Content Addressing + +Content addressing identifies data by the cryptographic hash of its contents rather than by location or arbitrary name, ensuring the same content produces the same identifier regardless of where it's stored. Mononoke uses Blake2b (256-bit) for hashing, while Git and Mercurial/Sapling use SHA-1. + +The blobstore is a key-value store that holds data with different addressing strategies, some but not all of which is content-addressed. Fully content-addressed data uses the hash of the value as the key, for example Bonsai changesets are stored with `ChangesetId` as the key, which is the Blake2b hash of the changeset contents. Logically content-addressed data uses a content hash as the key, but the stored value may have a different structure in the store, for example file contents use `ContentId` (the hash of the file) as the key, but large files are chunked for storage. Non-content-addressed data uses keys derived from other objects, for example blame data is keyed by the corresponding unode ID rather than by hashing the blame information. + +Content-addressed data provides immutability (any modification changes the hash), integrity verification (hashes serve as checksums), and deduplication (identical content produces identical keys). The Bonsai changeset graph forms a Merkle DAG where each changeset includes hashes of its parent changesets, making the repository history tamper-evident. + +## Repository Facets + +Mononoke uses a facet pattern to compose repository functionality. Rather than having a monolithic repository class with all possible methods, repository capabilities are broken into discrete "facets" that can be composed together. Each facet provides a specific capability—for example, `RepoIdentity` provides repository name and ID, `RepoBlobstore` provides access to immutable blob storage, and `CommitGraph` provides commit graph traversal operations. + +Functions declare their requirements explicitly by specifying which facets they need through trait bounds. This makes dependencies clear and enables better modularity and testability. The facet pattern also allows different repository types to mix and match capabilities as needed. Facets are defined in the `repo_attributes/` directory and are used throughout Mononoke's codebase as the standard way to access repository functionality. + +## Derived Data + +Derived data is computed information that can be regenerated from Bonsai changesets and file content blobs. While Bonsai represents the core source of truth, many common operations would be inefficient without precomputed indexes. Derived data types include manifests (directory structures), file history information, blame annotations, and VCS-specific formats needed for Git and Mercurial protocol compatibility. + +The key architectural decision is that derived data computation happens asynchronously, off the critical path of commit ingestion. When a new commit is pushed, only the Bonsai changeset and file contents are written synchronously. Derived data is then computed in the background, often by a separate derivation service that can scale horizontally. This separation allows Mononoke to maintain high write throughput even as more derived data types are added. Derived data is stored in the blobstore and can be backfilled or migrated independently of the core commit data. + +## Blobstore + +The blobstore is Mononoke's immutable key-value storage layer for repository data. It stores Bonsai changesets, file content blobs, derived data, and other repository artifacts. Each blob is identified by a unique key and, once written, is never modified. + +Mononoke uses a layered blobstore architecture built on the decorator pattern. Storage backends include fileblob (filesystem), sqlblob (MySQL/SQLite), s3blob (Amazon S3), and manifoldblob (Meta-internal). These backends are wrapped with decorators that add functionality: `cacheblob` provides multi-level caching (memcache and in-process cachelib), `multiplexedblob` enables writes to multiple backends for redundancy, `packblob` provides compression, and `redactedblobstore` enforces content redaction policies. This decorator stack allows Mononoke to compose complex storage behaviors from simple, reusable components. The immutability of blobstore data enables aggressive caching and simplifies consistency reasoning. + +## Metadata Database + +While the blobstore holds immutable content, the metadata database stores mutable repository state and indexes. This includes bookmarks (branch pointers), the commit graph index for efficient ancestry queries, and various mapping tables that connect Bonsai changesets to their external representations. + +The metadata database uses MySQL in production and SQLite for development and testing. Unlike blobstore operations, database operations can involve transactions and updates. For example, when a bookmark moves, the database is updated to point to the new changeset. The metadata database is also used for operational concerns like tracking cross-repository sync progress and managing derived data derivation state. The separation between immutable blobstore data and mutable database state is a fundamental architectural principle that allows Mononoke to scale writes while maintaining consistency. + +## VCS Mappings + +VCS mappings connect Mononoke's internal Bonsai representation to external version control system identities. When a Git commit or Mercurial changeset is imported into Mononoke, it is converted to Bonsai format, and a bidirectional mapping is stored. The `bonsai_git_mapping` table maps Bonsai changeset IDs to Git commit SHA-1 hashes, while `bonsai_hg_mapping` maps to Mercurial changeset hashes. + +These mappings are essential for serving Git and Mercurial clients. When a client requests a commit by its Git or Mercurial ID, Mononoke uses the mapping to find the corresponding Bonsai changeset, operates on the Bonsai representation internally, and then converts results back to the client's expected format. Additional mappings exist for other identifier schemes like globalrevs (sequential integers for SVN-style workflows) and svnrevs (for repositories imported from Subversion). The mapping tables are stored in the metadata database and enable Mononoke to present a consistent view of repository history regardless of which client type is being used. + +## Related Documentation + +* [Architecture Overview](1.3-architecture-overview.md) - How these concepts fit together in the overall system +* [Bonsai Data Model](2.1-bonsai-data-model.md) - Detailed exploration of Bonsai format +* [Repository Facets](2.2-repository-facets.md) - Complete facet pattern explanation +* [Derived Data](2.3-derived-data.md) - Framework and implementation details +* [Storage Architecture](2.4-storage-architecture.md) - Blobstore and database design diff --git a/eden/mononoke/docs/1.3-architecture-overview.md b/eden/mononoke/docs/1.3-architecture-overview.md new file mode 100644 index 0000000000000..0b636ee6944e2 --- /dev/null +++ b/eden/mononoke/docs/1.3-architecture-overview.md @@ -0,0 +1,570 @@ +# Architecture Overview + +This document provides a high-level overview of Mononoke's architecture. After reading this, you should understand how Mononoke's major components fit together, how data flows through the system, and the key architectural decisions that shape the codebase. + +**Reading time: ~30 minutes** + +## Introduction + +Mononoke is a distributed source control server designed to scale along multiple dimensions: commit rate, repository size, file count, and branch count. Rather than being a monolithic application, Mononoke is structured as a collection of Rust libraries that implement source control functionality, which are then composed into various servers, services, and tools. + +The architecture is designed around several core principles: + +1. **Stateless servers** - All persistent state lives in external storage (blobstore, databases) +2. **Horizontal scalability** - Servers can be scaled independently to handle load +3. **VCS independence** - A canonical data model supports multiple version control systems +4. **Separation of concerns** - Write path optimized separately from read path +5. **Modular composition** - Repository functionality built from composable facets + +## System Architecture: Service Composition + +Mononoke is designed as a distributed system composed of multiple services that work together. + +### Service Tiers + +Mononoke is organized into three tiers: + +#### 1. Frontend Services (Stateless) + +These services serve external clients and implement various protocols. They are stateless and can be horizontally scaled to handle client load. + +**Protocol Servers:** +- **Mononoke Server (SLAPI, formerly EdenAPI)** - Serves Sapling CLI and EdenFS clients using the SLAPI protocol over HTTP +- **Git Server** - Serves Git clients using the Git protocol over HTTP +- **LFS Server** - Serves Git LFS requests +- **SCS Server** - Provides a Thrift API for programmatic access to repositories + +All frontend services authenticate requests, translate between protocol-specific formats and Mononoke's internal Bonsai format, and route operations to the appropriate backend storage or microservices. + +#### 2. Internal Microservices + +These services handle expensive or serialized operations that are offloaded from the main frontend servers. This prevents resource contention and enables dedicated scaling for specific workloads. + +**Microservices:** +- **Land Service** - Handles commit landing (merging) operations, serializing writes to public bookmarks to prevent conflicts +- **Derived Data Service** - Coordinates asynchronous computation of derived data across multiple workers +- **Diff Service** - Computes diffs for code review and other tools +- **Bookmark Service** - Maintains warm caches of bookmark state + +By offloading these operations to dedicated services, frontend servers remain responsive to lightweight operations like fetches and clones. + +#### 3. Backend Storage + +All services (frontend and microservices) share common backend storage: + +**Storage Systems:** +- **Blobstore** - Immutable key-value storage for file contents, commit metadata, derived data, and manifests +- **Metadata Database** - Mutable SQL database storing bookmarks, VCS mappings, commit graph index, and repository state + +This shared storage enables stateless services and allows any server to handle any request for a given repository. + +### System Diagram + +[![Mononoke Architecture](px/8w0zr)](https://www.internalfb.com/excalidraw/EX271339) + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Client Layer │ +│ Sapling CLI │ EdenFS │ Git Clients │ Other Services │ +└──────────────────────────────────────────────────────────────┘ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Frontend Services (Stateless) │ +│ │ +│ Mononoke Server (SLAPI) │ Git Server │ LFS Server │ +│ SCS Server (Thrift) │ +└──────────────────────────────────────────────────────────────┘ + ▼ + ┌───────────────┴───────────────┐ + ▼ ▼ +┌────────────────────────┐ ┌──────────────────────────────┐ +│ Internal Microservices│ │ Backend Storage │ +│ │ │ │ +│ Land Service │◄────►│ Blobstore (immutable) │ +│ Derived Data Service │ │ - File contents │ +│ Diff Service │ │ - Commit metadata │ +│ Bookmark Service │ │ - Derived data │ +│ │ │ │ +│ │ │ Metadata DB (mutable) │ +│ │ │ - Bookmarks │ +│ │ │ - VCS mappings │ +│ │ │ - Commit graph │ +└────────────────────────┘ └──────────────────────────────┘ +``` + +### Service Architecture Characteristics + +This service-oriented architecture has the following characteristics: + +**Independent Scaling** - Each tier can be scaled independently. Git servers can be added to increase capacity for Git operations, while derivation workers can be scaled separately. + +**Workload Isolation** - Expensive operations (landing, derivation) run in dedicated services, isolating them from lightweight operations (fetches, file reads). + +**Sequential Coordination** - Operations requiring mutual exclusion (e.g., landing to the same bookmark) are coordinated by routing through a single service instance. + +**Independent Deployment** - Services can be deployed, updated, and monitored independently. + +**Resource Allocation** - Different services have different resource profiles (CPU, memory, I/O) and can be provisioned accordingly. + +### Relationship to Code Architecture + +It's important to understand that the service architecture and code architecture are orthogonal concerns: + +- **Service architecture** describes how Mononoke is deployed as a distributed system (frontend services, microservices, storage) +- **Code architecture** describes how each service/tool is built internally using shared libraries (common → facets → features → API) + +Every Mononoke binary - whether it's the SLAPI server, the land service, or the admin tool - uses the same internal layered structure. They all share the same repository implementation, blobstore abstractions, and core types. The difference is in what protocol they serve or what operations they perform. + +## Code Architecture: Internal Application Structure + +While the system architecture describes how Mononoke services are composed, each individual Mononoke application (whether a server, microservice, tool, or job) is structured internally using a consistent layered design. This code architecture is implemented as a collection of Rust libraries that are composed together. + +Understanding this internal structure is key to navigating the codebase and understanding how to build new tools or modify existing ones. + +### Application Layering + +Every Mononoke application - whether it's the main SLAPI server, the admin CLI tool, or the walker job - is built using the same layered architecture. This consistent structure makes it easier to understand any part of the codebase once you understand the pattern. + +The layers are: + +``` +┌──────────────────────────────────┐ +│ Application Binary │ (server, tool, or job main()) +│ (uses cmdlib/mononoke_app) │ +└──────────────────────────────────┘ + ▼ +┌──────────────────────────────────┐ +│ API Layer (optional) │ High-level abstractions +│ (mononoke_api) │ (Repo, Changeset, File objects) +└──────────────────────────────────┘ + ▼ +┌──────────────────────────────────┐ +│ Features │ High-level operations +│ (pushrebase, cross_repo_sync) │ (combines multiple facets) +└──────────────────────────────────┘ + ▼ +┌──────────────────────────────────┐ +│ Repository Attributes │ Composable capabilities +│ (repo_attributes/*) │ (facets providing specific functionality) +└──────────────────────────────────┘ + ▼ +┌──────────────────────────────────┐ +│ Common Components │ Foundation libraries +│ (blobstore, mononoke_types, │ (storage, types, utilities) +│ common/*, cmdlib) │ +└──────────────────────────────────┘ +``` + +Each layer is described below: + +### 1. Common Components (Foundation Layer) + +The foundation layer provides basic building blocks used by all Mononoke applications. These libraries know nothing about repositories or high-level version control concepts - they provide fundamental primitives. + +**Key components:** +- **Context (`context/`)** - Request context carrying permissions, logging, and tracing +- **Blobstore abstractions (`blobstore/`)** - Key-value storage interface and implementations +- **Common utilities (`common/`)** - Async utilities, SQL helpers, logging extensions +- **Core types (`mononoke_types/`)** - Fundamental data structures (Bonsai changesets, file content) +- **Configuration (`metaconfig/`)** - Repository configuration system +- **Permission checking (`permission_checker/`)** - Access control primitives + +These components are designed to be reusable and have minimal dependencies on each other. + +### 2. Repository Attributes (Composition Layer) + +Repository attributes are implemented as **facets** - trait-based components that provide specific capabilities. Each facet encapsulates a single responsibility and can contain state that forms part of the repository. + +**Core attribute categories:** + +**Identity and Configuration:** +- `repo_identity` - Repository ID and name +- `repo_config` - Repository configuration + +**Storage:** +- `repo_blobstore` - Access to immutable blob storage +- `filestore` - File content storage with chunking + +**Commit Graph:** +- `commit_graph` - Efficient ancestry queries and traversal +- `commit_graph_writer` - Write interface for graph updates + +**Derived Data:** +- `repo_derived_data` - Manager for all derived data types +- `repo_derivation_queues` - Remote derivation coordination + +**VCS Integration:** +- `bonsai_hg_mapping` - Bonsai ↔ Mercurial changeset mappings +- `bonsai_git_mapping` - Bonsai ↔ Git commit mappings +- `bonsai_globalrev_mapping` - Sequential integer IDs +- `git_symbolic_refs` - Git reference handling + +**Bookmarks and References:** +- `bookmarks` - Branch management (read/write) +- `bookmark_update_log` - History of bookmark movements +- `bookmarks_cache` - Cached bookmark data + +**Operations:** +- `repo_permission_checker` - Access control +- `hook_manager` - Pre-commit hooks +- `repo_cross_repo` - Cross-repository operations + +Facets are located in `repo_attributes/` and are composed together into complete repositories using the facet container pattern. This allows different operations to declare exactly which repository capabilities they need. + +### 3. Features (High-Level Operations) + +Features combine multiple facets to implement source control operations. Unlike facets, features don't hold state - they orchestrate operations across repository attributes. + +**Key features (in `features/`):** +- **Pushrebase (`features/pushrebase/`)** - Server-side rebasing for linear history +- **Cross-repo sync (`features/cross_repo_sync/`)** - Repository synchronization +- **Commit transformation (`features/commit_transformation/`)** - Rewriting commits +- **Redaction (`features/redaction/`)** - Content removal +- **Diff and history (`features/diff/`, `features/history_traversal/`)** - Content comparison and traversal +- **Bookmark movement (`bookmarks/bookmarks_movement/`)** - Safe bookmark updates +- **Microwave (`features/microwave/`)** - Cache warming + +Features are typically implemented as functions that accept trait bounds specifying the required facets. + +### 4. API Layer (Optional) + +The API layer (`mononoke_api/` and `mononoke_api_hg/`) provides high-level abstractions over the underlying implementation details. Not all applications use this layer - it's primarily used by servers and interactive tools. + +It exposes objects like: + +- **Repo** - Repository operations +- **Changeset** - Commit metadata and operations +- **File** - File content and history +- **Tree** - Directory listings + +This layer hides whether operations use commit graph vs. segmented changelog, which manifest type is used, etc. It provides a stable interface that isolates protocol implementations from internal representation changes. + +### 5. Application Framework (cmdlib) + +All Mononoke binaries use the `cmdlib/mononoke_app/` framework, which provides: + +- Argument parsing and configuration loading +- Monitoring and observability setup +- Graceful shutdown handling +- Repository initialization +- Common command-line options + +This ensures all Mononoke applications have consistent behavior, configuration, and operational characteristics. + +### How Applications Use These Layers + +Different types of applications use different subsets of these layers: + +**Servers** (Mononoke, Git, SCS): +``` +Server Binary (cmdlib/mononoke_app) + ↓ +API Layer (mononoke_api) + ↓ +Features (pushrebase, hooks) + ↓ +Repository Facets + ↓ +Common Components +``` + +**Tools** (admin CLI, walker): +``` +Tool Binary (cmdlib/mononoke_app) + ↓ +Features (or direct facet access) + ↓ +Repository Facets + ↓ +Common Components +``` + +**Microservices** (Land, Derived Data): +``` +Service Binary (cmdlib/mononoke_app) + ↓ +Features + ↓ +Repository Facets + ↓ +Common Components +``` + +The key insight is that all applications share the same library codebase. A function implementing pushrebase can be used by the SLAPI server (via API layer), the land service (directly), and the admin tool (for testing). + +## Data Model and Flow + +### The Bonsai Data Model + +At the heart of Mononoke is **Bonsai**, a VCS-agnostic data model that serves as the single source of truth. Bonsai is designed for: + +- **VCS independence** - Not tied to Git or Mercurial internals +- **High throughput writes** - Minimal metadata, optimized for fast commit ingestion +- **Content addressing** - Blake2b hashing creates a Merkle DAG ensuring data integrity +- **Multi-VCS support** - Can be converted to/from Git and Mercurial formats + +**Core Bonsai data includes:** +- **Bonsai changesets** - Commit metadata (parents, author, date, message, file changes) +- **File content blobs** - The actual file contents, stored with content-based keys enabling deduplication + +Everything else in Mononoke is derived from or maps to this core data. + +### Write Path vs. Read Path + +Mononoke's architecture separates the **write path** (committing new data) from the **read path** (serving queries), optimizing each independently. + +#### Write Path: Core Data Only + +When a commit is pushed: + +1. **Validate and transform** - Client's commit (Git/Mercurial format) is validated +2. **Create Bonsai changeset** - Converted to canonical Bonsai format +3. **Store in blobstore** - Bonsai changeset and new file contents written +4. **Update metadata DB** - VCS mapping (Bonsai ↔ Git/Mercurial hash) stored +5. **Update commit graph** - Parent relationships indexed +6. **Run hooks** - Pre-commit policy enforcement +7. **Move bookmark** - Branch pointer updated +8. **Push completes** - Next commit can now proceed + +Derived data is not computed on the write path. This keeps the critical section minimal and push latency low. + +#### Read Path: Derived Data + +Most read operations require data not present in the minimal Bonsai format. For example: +- **Mercurial clients** need Mercurial manifests and filenodes +- **Git clients** need Git trees and commits +- **Blame** needs line-by-line author attribution +- **File listings** need efficient directory traversal + +These are provided by **derived data** - indexes and representations computed asynchronously from Bonsai changesets. Derived data is: + +- **Computed off the critical path** - After push completes +- **Stored in blobstore** - Cached for future reads +- **Dependency-aware** - Some types depend on others (blame depends on unodes) +- **Derivable in parallel** - For independent changesets and types +- **Versionable** - Can be redesigned and recomputed (backfilled) + +**Major derived data types:** +- **Manifests** - Directory structures (fsnodes, unodes, skeleton manifests, Git trees) +- **File metadata** - Filenodes (Mercurial), blame, fastlog +- **Git-specific** - Git commits, delta manifests +- **Utilities** - Changeset info, deleted manifest, case conflict detection + +The derived data service can coordinate derivation across workers, deduplicating work and enabling horizontal scaling of derivation workload. + +### Request Flow Example: Fetching a File + +A request flows through the system as follows: + +1. **Client request** - Sapling client requests file content via SLAPI +2. **Protocol server** - Mononoke server authenticates and translates request +3. **API layer** - Calls high-level file fetch API +4. **Facets** - API uses: + - `repo_derived_data` to get appropriate manifest type + - `repo_blobstore` to fetch manifest and file content + - `bonsai_hg_mapping` (if needed) to resolve Mercurial hash to Bonsai +5. **Storage layer** - Blobstore returns data (from cache or backend) +6. **Response** - Data flows back through layers to client + +Throughout this flow, caching (cachelib, memcache) reduces load on storage backends. + +## Storage Architecture + +Mononoke uses two complementary storage systems: + +### Immutable Blobstore + +The blobstore is a key-value store for immutable data: + +- **File contents** - Actual file data, chunked for large files +- **Bonsai changesets** - Commit metadata +- **Derived data** - Manifests, blame, filenodes, etc. +- **VCS-specific formats** - Original Git/Mercurial bytes + +**Blobstore implementations:** +- **Manifoldblob** - Primary production backend (Facebook-internal) +- **SQLblob** - SQL database backend +- **S3blob** - Amazon S3 backend +- **Fileblob** - Filesystem (development/testing) +- **Memblob** - In-memory (testing) + +**Blobstore decorator pattern:** + +Mononoke uses decorators to layer functionality: + +``` +Application + ↓ +prefixblob (repo-specific key namespacing) + ↓ +cacheblob (memcache + cachelib caching) + ↓ +multiplexedblob (write-all, read-any across backends) + ↓ +packblob (compression for space efficiency) + ↓ +Backend storage (sqlblob/manifoldblob/s3blob) +``` + +Additional decorators include redaction (content removal), sampling (observability), throttling (rate limiting), and chaos/delay (testing). + +**Multiplexing for availability:** + +To avoid cyclic dependencies (storage systems need source control to deploy fixes), Mononoke writes to multiple independent blobstores simultaneously. If one backend is down, reads succeed from others. A healer job ensures consistency across backends. + +### Mutable Metadata Database + +A SQL database (MySQL in production, SQLite for development) stores mutable repository state and mappings: + +- **Bookmarks** - Current branch positions +- **VCS mappings** - Bonsai ↔ Git/Mercurial/SVN/Globalrev hash mappings +- **Commit graph index** - Parent/child relationships for fast queries +- **Phases** - Draft vs. public commit status +- **Bookmark update log** - Audit trail of all bookmark movements +- **Counters** - Mutable counters for sync operations + +The metadata DB is used for small, frequently changing data, while the blobstore handles large, immutable content. + +### Caching Strategy + +Mononoke employs multi-level caching: + +1. **In-memory cache (Cachelib)** - Per-server process cache +2. **Shared cache (Memcache)** - Cross-server cache within a region +3. **Warm bookmark cache** - Precomputed state for important branches + +Cache configuration can be controlled via command-line flags (e.g., `--cache-mode=local-only`). + +## Architectural Characteristics + +Several design choices shape Mononoke's implementation: + +**VCS-Agnostic Data Model** - Mononoke uses Bonsai as its internal representation rather than Git or Mercurial formats. This requires conversion when serving clients but allows a single backend to support multiple version control systems. VCS mappings maintain bidirectional relationships between Bonsai and external commit identifiers. + +**Asynchronous Derivation** - Derived data (manifests, blame, filenodes) is computed off the critical write path. When a commit is pushed, only the Bonsai changeset and file contents are written synchronously. Indexes are computed asynchronously by derivation workers. This reduces push latency but introduces eventual consistency for derived data. + +**Facet Composition** - Repository functionality is provided through composable facets (traits) rather than monolithic objects. Each facet encapsulates a specific capability such as blobstore access, commit graph traversal, or bookmark management. Functions declare their requirements through trait bounds, making dependencies explicit. + +**Stateless Services** - Servers maintain no persistent state. All repository data resides in external storage (blobstore and metadata database). This enables horizontal scaling and simplifies deployments but requires caching for performance. + +**Decorator-Based Storage** - Blobstore functionality is composed through decorators that add caching, multiplexing, compression, and other capabilities. This allows flexible configuration of the storage stack at the cost of additional abstraction layers. + +**Microservice Delegation** - Expensive operations (landing, large diffs, coordinated derivation) are delegated to dedicated microservices. This isolates resource-intensive work from lightweight read operations and enables serialization of operations that require mutual exclusion. + +## Putting It All Together + +Understanding Mononoke's architecture requires understanding both the system architecture (how services are composed) and the code architecture (how each application is structured internally). + +### Service Composition + +At the system level, requests flow through the service tiers: + +**Read Request (e.g., fetch a file):** +``` +Client (Sapling) + → Frontend Service (Mononoke Server) + → Backend Storage (Blobstore + Metadata DB) + → Response +``` + +**Write Request (e.g., push commits):** +``` +Client (Sapling) + → Frontend Service (Mononoke Server) + → Microservice (Land Service) + → Backend Storage (Blobstore + Metadata DB) + → Microservice (Derived Data Service, async) + → Response +``` + +The frontend services are stateless and can scale horizontally. Microservices handle expensive or serialized operations. Backend storage is shared by all services. + +### Code Layer Dependencies + +Within each application, dependencies flow downward through the library layers: + +``` +Application Binaries (servers, tools, jobs) + ↓ use +API Layer (mononoke_api) - optional, for high-level abstractions + ↓ uses +Features (pushrebase, cross_repo_sync, hooks) + ↓ use +Repository Facets (repo_attributes/*) + ↓ use +Common Components (blobstore, mononoke_types, common/*) +``` + +Lower layers have no knowledge of upper layers. This keeps dependencies clean and makes testing easier. + +### Data Flow Patterns + +Understanding how data flows helps clarify the separation of concerns: + +**Push (Write Path):** +``` +Client → Protocol Server → Hooks → Bonsai Creation → + Blobstore Write → Metadata DB Update → Bookmark Move → + Async Derivation (off critical path) +``` + +**Pull (Read Path):** +``` +Client → Protocol Server → API Layer → + Check Derived Data → Fetch from Blobstore (via cache) → + Convert to VCS Format → Return to Client +``` + +**Background Derivation:** +``` +Derivation Worker → Check What Needs Derivation → + Fetch Dependencies → Compute Derived Data → + Store in Blobstore → Update Derivation Status +``` + +### Directory Organization + +The Mononoke codebase is organized to reflect both architectures: + +**Library Layers (Code Architecture):** +- `common/` - Foundation components (utilities, async helpers) +- `blobstore/` - Storage abstractions and implementations +- `mononoke_types/` - Core data types (Bonsai, content IDs) +- `repo_attributes/` - Repository facets (~35 composable capabilities) +- `features/` - High-level operations (pushrebase, cross-repo sync, etc.) +- `mononoke_api/` - API layer abstractions +- `cmdlib/` - Application framework + +**Applications (System Architecture):** + +*Frontend Services:* +- `server/` - Mononoke Server (SLAPI) +- `git_server/` - Git protocol server +- `lfs_server/` - LFS protocol server +- `scs/` - SCS Thrift API server + +*Microservices:* +- `servers/land_service/` - Landing operations +- `derived_data/remote/` - Derived data coordination +- Diff service, bookmark service (various locations) + +*Tools and Jobs:* +- `tools/admin/` - Admin CLI +- `tools/testtool/` - Testing utilities +- `jobs/walker/` - Graph walker +- `jobs/blobstore_healer/` - Storage healer + +This organization makes it clear which directories contain reusable libraries (used by all applications) and which contain application-specific code (servers, tools, jobs). + +## Next Steps + +This overview provides the big picture of Mononoke's architecture. To dive deeper: + +- **[Key Concepts](1.2-key-concepts.md)** - Essential terminology and concepts +- **[Navigating the Codebase](1.4-navigating-the-codebase.md)** - Finding your way around approximately 55-60 directories +- **[Bonsai Data Model](2.1-bonsai-data-model.md)** - Deep dive into the core data model +- **[Repository Facets](2.2-repository-facets.md)** - Understanding the facet pattern +- **[Derived Data](2.3-derived-data.md)** - How indexes are computed and managed +- **[Storage Architecture](2.4-storage-architecture.md)** - Blobstore and database details + +Component-specific documentation lives in the respective directories alongside the code. diff --git a/eden/mononoke/docs/1.4-navigating-the-codebase.md b/eden/mononoke/docs/1.4-navigating-the-codebase.md new file mode 100644 index 0000000000000..acfe1c3ac79ce --- /dev/null +++ b/eden/mononoke/docs/1.4-navigating-the-codebase.md @@ -0,0 +1,522 @@ +# Navigating the Mononoke Codebase + +This guide helps developers find their way around Mononoke's approximately 55-60 top-level directories. Rather than providing an exhaustive catalog, this document teaches the organizational principles and patterns that make the codebase navigable. + +## Overview + +Mononoke's codebase is organized around two key architectural concepts explained in the [Architecture Overview](1.3-architecture-overview.md): + +1. **System Architecture** - How Mononoke is deployed as frontend services, microservices, and shared storage +2. **Code Architecture** - How each application is built internally using layered libraries + +This document focuses on the **code architecture** and directory organization. The library code is organized into architectural layers, with each layer building on the ones below it. Understanding this layered structure is key to knowing where to find (or add) functionality: + +``` +Servers & Tools (Entry points for users and operators) + ↓ + API Layer (High-level source control abstractions) + ↓ + Features Layer (Source control operations) + ↓ +Repo Attributes Layer (Repository facets and capabilities) + ↓ +Base Components Layer (Fundamental building blocks) +``` + +This layering principle guides the directory organization. Lower layers know nothing about higher layers - for example, base components know nothing about repositories, and repository attributes don't implement complete features. + +## Directory Organization Principles + +### Two-Level Hierarchy + +Almost all components follow a two-level structure: +- **Category directory**: Groups related components (e.g., `blobstore/`, `repo_attributes/`) +- **Component directory**: Contains the actual implementation + +Each component directory typically contains: +- `src/` - Rust source files +- `BUCK` - Build definitions +- Optional: `Cargo.toml` for OSS builds + +### Flat Namespace for Crates + +Rust uses a flat namespace for crate names, so Mononoke components use descriptive names to avoid collisions. For example, the fsnodes component is named `fsnodes_derivation` rather than just `fsnodes`, and SQL storage implementations often use database-related prefixes (e.g., `dbbookmarks`, `sql_commit_graph_storage`). + +### Oncall Specification + +All BUCK files start with `oncall("scm_server_infra")` to designate ownership. This is standard across the entire codebase. + +## Finding Components by Layer + +### Base Components (Foundation) + +The lowest layer provides fundamental building blocks that know nothing about repositories or source control concepts. + +**`common/`** - Shared utilities and libraries +- Async utilities: `futures_watchdog`, `async_limiter` +- Logging: `scuba_ext`, `logger_ext` +- Data structures: `dedupmap`, `uniqueheap` +- Time and measurement: `time_measuring`, `reloader` +- Graph algorithms: `topo_sort` +- And many more specialized utilities + +**`blobstore/`** - Immutable key-value storage +- Backend implementations: `fileblob/`, `memblob/`, `sqlblob/`, `s3blob/` +- Caching: `cacheblob/` +- Multi-backend: `multiplexedblob/`, `multiplexedblob_wal/` +- Compression: `packblob/` (see its README.md for details) +- Storage decorators: `redactedblobstore/`, `throttledblob/`, `prefixblob/` +- Testing utilities: `chaosblob/`, `delayblob/` +- Ephemeral storage: `ephemeral_blobstore/` + +**`mononoke_types/`** - Core data type definitions +- Defines Bonsai changesets, file changes, paths, and content IDs +- See `mononoke_types/docs/` for detailed type documentation +- This is monolithic but could be split up in the future + +**`cmdlib/`** - Command-line application framework +- `mononoke_app/` - Standard framework for new binaries (use this!) +- `base_app/` - Base application primitives +- Argument handling: `config_args/`, `commit_id/`, `sharding/` +- Environment setup: `environment/`, `logging/`, `log/` +- Common capabilities: `caching/`, `scrubbing/`, `cross_repo/` + +These directories provide the foundation but contain no repository-specific logic. + +### Repository Attributes (Facets) + +The `repo_attributes/` directory contains all the facets that compose a repository. Each subdirectory implements one aspect of repository functionality. + +**Finding facets**: All repository attributes live in `repo_attributes/`. If you need to understand or modify a specific repository capability, start here. + +Key facet categories: + +**Identity and Configuration** +- `repo_identity/` - Repository name and ID +- `repo_bookmark_attrs/` - Bookmark configuration attributes + +**Storage Access** +- `repo_blobstore/` - Repository-specific blobstore access +- `filestore/` - File content storage and retrieval +- `mutable_blobstore/` - Mutable blob operations + +**Commit Graph and History** +- `commit_graph/` - Contains `commit_graph/` for reading and `sql_commit_graph_storage/` for storage +- `phases/` - Commit phase tracking (public, draft, etc.) + +**Derived Data** +- `repo_derived_data/` - Per-repository derived data management +- `repo_derivation_queues/` - Derivation work queues + +**VCS Mappings** +- `bonsai_hg_mapping/` - Bonsai ↔ Mercurial changeset mapping +- `bonsai_git_mapping/` - Bonsai ↔ Git commit mapping +- `bonsai_globalrev_mapping/` - Bonsai ↔ GlobalRev mapping +- `bonsai_svnrev_mapping/` - Bonsai ↔ SVN revision mapping +- `bonsai_tag_mapping/` - Tag object mappings +- `bonsai_blob_mapping/` - Blob content mappings + +**Bookmarks (Branches)** +- `bookmarks/` - Contains `bookmarks/` for bookmark access and `dbbookmarks/` for storage + +**Git-Specific Attributes** +- `git_ref_content_mapping/` - Git reference to content mappings +- `git_source_of_truth/` - Tracks Git source of truth +- `git_symbolic_refs/` - Git symbolic reference handling + +**Operations and Metadata** +- `hook_manager/` - Hook execution management +- `pushrebase_mutation_mapping/` - Pushrebase mutation tracking +- `deletion_log/` - Deleted commit tracking +- `mutable_counters/` - Repository counters +- `mutable_renames/` - File rename tracking +- `repo_cross_repo/` - Cross-repository sync attributes +- `repo_lock/` - Repository locking +- `repo_permission_checker/` - Permission checking +- `repo_sparse_profiles/` - Sparse profile management +- `restricted_paths/` - Path access restrictions +- `sql_query_config/` - SQL query configuration +- `repo_metadata_checkpoint/` - Metadata checkpointing +- `repo_event_publisher/` - Event publishing + +**Legacy Filenode Storage** +- `filenodes/` - Legacy filenode interface +- `newfilenodes/` - Newer filenode implementation + +**Pattern for using facets**: Features and higher layers access repository capabilities through facet traits. Look at the dependencies in a feature's BUCK file to see which facets it uses. + +### Features Layer + +The `features/` directory contains source control operations implemented by combining repository facets. Features are stateless - they operate on facets but don't hold repository state themselves. + +Current features in `features/`: +- `async_requests/` - Asynchronous request handling +- `cache_warmup/` - Repository cache preloading +- `changesets_creation/` - Changeset creation operations +- `commit_cloud/` - Commit cloud synchronization +- `commit_rewriting/` - Commit transformation and rewriting +- `commit_transformation/` - Commit rewriting and transformation +- `cross_repo_sync/` - Cross-repository synchronization +- `diff/` - Diff computation +- `history_traversal/` - History walking and traversal +- `hooks/` - Pre-commit and other hooks +- `microwave/` - Fast cache warming +- `pushrebase/` - Server-side rebasing +- `redaction/` - Content redaction +- `repo_metadata/` - Repository metadata operations +- `repo_stats_logger/` - Repository statistics logging +- `repo_update_logger/` - Repository update logging + +**Other feature locations**: Some feature-related functionality is in other locations: +- `bookmarks/bookmarks_movement/` - Bookmark updates (within repo_attributes) + +The features layer is being gradually populated as code is refactored. + +### Derived Data + +The `derived_data/` directory contains the derived data framework and all derived data type implementations. + +**Framework Components** +- `manager/` - Derived data manager and coordination +- `remote/` - Remote derivation service +- `bulk_derivation/` - Batch derivation +- `constants/` - Shared constants +- `test_utils/` - Testing utilities + +**Derived Data Types** (partial list - there are ~22 total) +- **Manifests**: `fsnodes/`, `unodes/`, `skeleton_manifest/`, `skeleton_manifest_v2/`, `basename_suffix_skeleton_manifest_v3/`, `deleted_manifest/`, `case_conflict_skeleton_manifest/`, `content_manifest_derivation/` +- **File Metadata**: `filenodes_derivation/`, `blame/`, `fastlog/` +- **Mercurial-specific**: `mercurial_derivation/` +- **Utilities**: `changeset_info/`, `inferred_copy_from/` +- **Testing**: `test_manifest/`, `test_sharded_manifest/` + +**Finding derived data types**: All implementations are subdirectories of `derived_data/`. Each type has its own directory with source and BUCK file. + +### API Layer + +The `mononoke_api/` directory provides high-level abstractions over Mononoke's internal data structures: +- Repository objects +- Changeset abstractions +- Tree and file interfaces +- VCS-agnostic operations + +There's also `mononoke_api_hg/` for Mercurial-specific API extensions. + +**Note**: The API layer is intended to be the primary interface for servers and tools, though many components still access lower layers directly for historical reasons. + +### Servers, Jobs, and Tools + +These directories contain the application binaries that use the library layers described above. As explained in the [Architecture Overview](1.3-architecture-overview.md), Mononoke is deployed as a collection of services (frontend servers and microservices) plus tools and background jobs. Each application uses the same layered library code. + +#### Servers + +Servers provide various protocols for accessing Mononoke. Each server has its own top-level directory: + +**Frontend Servers** (serve external clients) +- `server/` - Main Mononoke server (SLAPI for Sapling and EdenFS clients) + - Contains: `repo_listener/`, `context/`, `qps/` subdirectories + - The actual binary is defined in the top-level `BUCK` file +- `scs/` - Source Control Service (Thrift API for programmatic access) + - Contains: `scs_server/` for the server and `if/` for Thrift definitions +- `git_server/` - Git protocol server (HTTP-based) +- `lfs_server/` - Git LFS protocol server + +**Microservices** (handle expensive operations) +- `servers/land_service/` - Landing (merge) service +- `derived_data/remote/` - Remote derivation service (within derived_data) + +**Protocol Handlers** (libraries used by servers) +- `wireproto_handler/` - Mercurial wire protocol handling +- `edenapi_service/` - SLAPI service implementation +- `sshrelay/` - SSH relay server +- `hgproto/` - Mercurial protocol definitions + +#### Jobs (Background Workers) + +The `jobs/` directory contains long-running background jobs for maintenance and async operations: + +- `walker/` - Graph traversal, validation, scrubbing (see `walker/src/README.md`) +- `blobstore_healer/` - Storage durability and repair +- `cas_sync/` - Content-addressed storage synchronization +- `modern_sync/` - Modern sync job +- `statistics_collector/` - Repository statistics collection + +Jobs are distinct from servers (which handle client requests) and tools (which are run on-demand). + +#### Tools (Command-Line Utilities) + +The `tools/` directory contains command-line utilities for operators and developers: + +**Administrative Tools** +- `admin/` - Main admin CLI (primary operational tool) +- `testtool/` - Testing and debugging utility + +**Import/Export** +- `blobimport/` - Import Mercurial repositories +- `import/` - Import utilities +- `repo_import/` - Repository import tool + +**Verification** +- `aliasverify/` - Verify content-addressed aliases +- `bonsai_verify/` - Bonsai changeset verification +- `check_git_wc/` - Git working copy verification + +**Maintenance** +- `packer/` - Packblob utilities +- `sqlblob_gc/` - SQL blobstore garbage collection +- `backfill_mapping/` - Backfill mapping tables + +**Other** +- `streaming_clone/` - Streaming clone generation +- `executor/` - Task executor +- `example/` - Example tool +- `tail-to-cloudwatch/` - CloudWatch log tailing + +The `admin/` tool is the primary interface for most operational tasks. Other tools are more specialized. + +#### Clients + +The `clients/` directory contains client tools and libraries for interacting with Mononoke servers: + +- `clients/scsc/` - Source Control Service CLI client (scsc command-line tool) +- `clients/git_pushrebase/` - Git pushrebase client utilities +- `clients/facebook/` - Facebook-internal client implementations + +These client tools provide command-line and programmatic interfaces to Mononoke's various services. + +### VCS Integration + +**Git Support** - `git/` directory +- `git_types/` - Git-specific type definitions +- `protocol/`, `packfile/`, `packetline/` - Git protocol implementation +- `gitimport/`, `gitexport/` - Import and export +- `import_direct/`, `import_tools/` - Import utilities +- `git_env/` - Git environment setup +- `bundle_uri/` - Bundle URI support +- `git-pool/` - Git object pooling +- `check_git_wc/` - Working copy checking + +**Mercurial Support** - `mercurial/` directory +- Mercurial type definitions +- Revlog support +- Legacy compatibility + +**LFS Support** +- `lfs_protocol/` - LFS protocol definitions +- `lfs_server/` - LFS server implementation +- `lfs_import_lib/` - LFS import utilities + +### Configuration and Metadata + +- `metaconfig/` - Repository metadata configuration (legacy name, should be `config/`) +- `mononoke_configs/` - Global configuration system +- `mononoke_macros/` - Rust procedural macros + +### Testing + +**Integration Tests** - `tests/integration/` +- `.t` test files (Mercurial-style test format) +- Test fixtures in `tests/fixtures/` +- Library scripts: `library.sh`, `library-commit.sh`, `library-git-lfs.sh`, etc. +- BUCK file defines `dott_test` targets for each test suite +- See `tests/integration/README.md` for comprehensive documentation + +**Test Utilities** - `tests/utils/` +- Shared testing utilities + +**Unit Tests**: Embedded in each component's `src/` directory (standard Rust practice) + +### Other Key Directories + +**Operations** +- `observability/` - Observability infrastructure +- `rate_limiting/` - Rate limiting implementations +- `permission_checker/` - Permission checking (base component) +- `repo_authorization/` - Repository authorization + +**Additional Components** +- `blobrepo/`, `blobrepo_utils/` - Legacy repository abstraction (being phased out in favor of facets) +- `repo_factory/` - Repository creation and initialization +- `manifest/` - Manifest handling utilities +- `megarepo_api/` - Megarepo (monorepo) operations +- `acl_regions/` - ACL region management +- `cas_client/` - Content-addressed storage client +- `cats/` - Configuration as a Service integration +- `repo_client/` - Repository client implementation +- `gotham_ext/` - Gotham web framework extensions +- `quiet_stream/` - Stream utilities +- `time_window_counter/` - Time-windowed counters +- `adaptive_rate_limiter/` - Adaptive rate limiting +- `benchmarks/` - Performance benchmarks +- `third_party/` - Third-party code + +**Facebook-Internal**: `facebook/` contains Facebook-specific implementations mirroring the main hierarchy. + +**OSS Builds**: `public_autocargo/` contains auto-generated Cargo.toml files for open-source builds. + +## Finding Specific Functionality + +### How to Find a Facet Implementation + +1. All facets are in `repo_attributes/` +2. Facet names are descriptive (e.g., `repo_identity`, `bookmarks`, `commit_graph`) +3. SQL storage for a facet is typically in a `sql_*` subdirectory within the facet + +Example: To find bookmark implementation: +- Facet interface: `repo_attributes/bookmarks/src/` +- SQL storage: Look for `dbbookmarks` within the bookmarks directory + +### How to Find a Derived Data Type + +1. All derived data types are in `derived_data/` +2. Directory names match the derived data type name +3. The manager and framework are in `derived_data/manager/` and `derived_data/remote/` + +Example: To find fsnodes derivation: +- Implementation: `derived_data/fsnodes/` +- Manager integration: Check `derived_data/manager/` for registration + +### How to Find Tests + +**Unit tests**: Look in the component's source directory +- Tests are co-located with code in `src/` files +- May also be in component-specific `test/` directories + +**Integration tests**: Look in `tests/integration/` +- Test files use `.t` extension +- Named descriptively (e.g., `test-pushrebase.t`, `test-gitimport.t`) +- Organized by functionality in BUCK file `dott_test` targets + +Example: To find pushrebase tests: +- Unit tests: `features/pushrebase/src/` may contain `#[test]` functions +- Integration tests: `tests/integration/test-pushrebase*.t` + +### How to Find a Server's Implementation + +Servers are at the top level, but the actual binary definition may be in the root BUCK file: + +- Server code: `server/`, `scs/`, `git_server/`, `lfs_server/` +- Binary definitions: Check top-level `BUCK` file or component BUCK files +- Service implementation: Often in a separate `*_service` directory (e.g., `edenapi_service/`) + +### How to Find Storage Implementation + +**Blobstore backends**: All in `blobstore/` with descriptive names +- File-based: `blobstore/fileblob/` +- SQL-based: `blobstore/sqlblob/` +- S3-based: `blobstore/s3blob/` +- In-memory: `blobstore/memblob/` + +**Metadata storage**: Look for `sql_*` subdirectories within facets +- Bookmarks: `repo_attributes/bookmarks/` area +- Commit graph: `repo_attributes/commit_graph/sql_commit_graph_storage/` +- Phases: Within `repo_attributes/phases/` + +## BUCK File Patterns + +BUCK files follow consistent patterns: + +``` +load("@fbsource//tools/build_defs:rust_library.bzl", "rust_library") + +oncall("scm_server_infra") + +rust_library( + name = "component_name", + srcs = glob(["src/**/*.rs"]), + autocargo = {"cargo_toml_dir": "component_name"}, # For OSS + deps = [ + # Dependencies listed here + ], +) +``` + +**Key patterns**: +- `oncall("scm_server_infra")` appears in every file +- `autocargo` metadata enables OSS builds +- `glob(["src/**/*.rs"])` is standard for sources +- Dependencies use full paths (e.g., `"//eden/mononoke/mononoke_types:mononoke_types"`) +- Third-party deps: `"fbsource//third-party/rust:crate_name"` + +**Finding dependencies**: Look at the `deps` field in a component's BUCK file to see what it uses. + +**Integration tests**: Use `dott_test` targets that reference test files and their binary dependencies. + +## Rules of Thumb + +1. **Repository capabilities?** → Look in `repo_attributes/` +2. **Source control operation?** → Check `features/`, then top-level directories +3. **Storage backend?** → Look in `blobstore/` +4. **Derived data type?** → Look in `derived_data/` +5. **Server implementation?** → Top-level server directories +6. **Background job?** → `jobs/` directory +7. **CLI tool?** → `tools/` directory (especially `tools/admin/`) +8. **VCS-specific code?** → `git/` or `mercurial/` directories +9. **Common utility?** → `common/` directory +10. **Test?** → `tests/integration/` for integration, `src/` for unit tests +11. **Configuration?** → `metaconfig/` or `mononoke_configs/` +12. **Unknown?** → Check the README.md in likely directories, or use grep/search + +## Common Navigation Scenarios + +### "I need to modify bookmark behavior" + +1. Facet interface: `repo_attributes/bookmarks/` +2. Bookmark updates: `repo_attributes/bookmarks/bookmarks_movement/` +3. Caching: `repo_attributes/bookmarks/bookmarks_cache/`, `repo_attributes/bookmarks/warm_bookmarks_cache/` +4. Storage: Look for SQL implementation within the bookmarks directory structure + +### "I need to add a new server endpoint" + +1. Identify the server: `server/`, `scs/`, `git_server/`, etc. +2. Check protocol service: `edenapi_service/`, `wireproto_handler/`, etc. +3. Look at similar endpoints in that server's source +4. Add integration tests in `tests/integration/` + +### "I need to add a new derived data type" + +1. Create new directory in `derived_data/` +2. Look at existing types (e.g., `derived_data/fsnodes/`) as examples +3. Implement derivation logic +4. Register with framework in `derived_data/manager/` +5. Add tests + +### "I'm debugging a storage issue" + +1. Identify the layer: blobstore or metadata database? +2. Blobstore: `blobstore/` + check decorators like `cacheblob/`, `multiplexedblob/` +3. Metadata: Find the facet in `repo_attributes/` and its SQL implementation +4. Check `walker/` for validation tools +5. Check `jobs/blobstore_healer/` for healing logic + +### "I need to understand a protocol" + +1. Git: `git/protocol/`, `git/packfile/`, `git/packetline/` +2. Mercurial: `hgproto/`, `wireproto_handler/` +3. SLAPI: `edenapi_service/` +4. LFS: `lfs_protocol/`, `lfs_server/` +5. SCS: `scs/if/source_control.thrift` for interface definition + +## Documentation Locations + +- **High-level docs**: `docs/` (this document and architecture docs) +- **Component-specific docs**: In component directories (e.g., `walker/src/README.md`, `blobstore/packblob/README.md`) +- **Type documentation**: `mononoke_types/docs/` +- **Integration test docs**: `tests/integration/README.md` + +## Summary + +Mononoke's directory structure follows consistent principles: +- **Layered architecture**: Base components → Repo attributes → Features → API → Servers/Tools +- **Two-level hierarchy**: Category directories contain component subdirectories +- **Descriptive naming**: Component names clearly indicate their purpose +- **Consistent patterns**: BUCK files, source layout, and testing follow standards + +When navigating the codebase: +- Start with the layer that matches your concern +- Use directory names as guides - they're descriptive +- Check README.md files in major directories +- Look at BUCK files to understand dependencies +- Follow the patterns established in existing code + +The codebase is large, but its organization is systematic. Understanding the layers and patterns makes it navigable. diff --git a/eden/mononoke/docs/2.1-bonsai-data-model.md b/eden/mononoke/docs/2.1-bonsai-data-model.md new file mode 100644 index 0000000000000..3d2958b1d157a --- /dev/null +++ b/eden/mononoke/docs/2.1-bonsai-data-model.md @@ -0,0 +1,321 @@ +# Bonsai Data Model + +This document explains Mononoke's core data model—the Bonsai format—which serves as the canonical, VCS-agnostic representation of repository data. Understanding Bonsai is essential for working with Mononoke, as it forms the foundation upon which all other repository operations are built. + +## Introduction + +Bonsai is Mononoke's internal representation of version control data. Every commit in Mononoke, regardless of whether it originated from Git, Mercurial, or Sapling, is stored internally as a Bonsai changeset. This unified format allows Mononoke to maintain a single source of truth while serving multiple client types. + +## Inherent Data vs. Derived Data + +Mononoke distinguishes between two fundamental categories of data: + +**Inherent data** constitutes the core Merkle DAG and serves as the source of truth. This data forms the basis for content-addressed hashing and must be sufficient to represent the entire state and history of the repository. In Mononoke, inherent data consists of: + +- Bonsai changesets +- File content blobs +- Original VCS bytes (for Git and Mercurial commits, preserved for compatibility) + +**Derived data** comprises indexes and representations computed from inherent data. This data is not included in content-addressed hashes and can be regenerated from inherent data. Derived data enables efficient operations that would be impractical using only the minimal Bonsai representation. + +This separation is fundamental to Mononoke's architecture. The write path stores only inherent data, keeping the critical section minimal. Derived data is computed asynchronously off the critical path, allowing Mononoke to maintain high write throughput while still providing efficient read operations. + +## Bonsai Changesets + +A Bonsai changeset represents a single commit. The structure is defined in `mononoke_types/src/bonsai_changeset.rs` and contains the following components: + +### Metadata Fields + +#### Parents +- A list of parent changeset identifiers + +#### Author Information +- Author name and email +- Author date (when the commit was authored) + +#### Committer Information +- Committer name and email (optional, used primarily for Git compatibility) +- Committer date (optional, may differ from author date in Git workflows) + +#### Commit Message +- The textual description of the change + +#### VCS-Specific Extra Fields +- `hg_extra`: Key-value pairs for Mercurial-specific metadata +- `git_extra_headers`: Key-value pairs for Git-specific headers +- `git_tree_hash`: Not used (deprecated) +- `git_annotated_tag`: Optional Git annotated tag information + +These fields allow Bonsai to preserve the complete semantics of both Git and Mercurial commits while maintaining a unified structure. + +### File Changes + +File changes are represented as a flat list of path-to-change mappings. Each entry specifies the full path to a file and the change that occurred. This differs from Git's nested tree structure and Mercurial's manifest system. + +Changes can be: + +**Tracked Changes** +- File additions or modifications +- Includes content identifier, file type, size, and optional copy-from information +- Copy-from information explicitly records file copies and renames, referencing the source path and changeset +- Git LFS field controls whether the file should be served as a Git LFS pointer when accessed via Git protocol + +**Tracked Deletions** +- Records that a file was removed + +**Untracked Changes and Deletions** +- Used for snapshot commits and working directory state that is not part of normal history + +#### File Type Information +- Regular file +- Executable file +- Symbolic link +- Git submodule + +The file change types are defined in `mononoke_types/src/file_change.rs`. + +### Snapshot and Subtree Support + +**Snapshot State** +- Indicates whether this changeset represents a snapshot of working directory state +- Snapshots may include untracked changes + +**Subtree Changes** +- Represent metadata for copies or merges that apply to subtrees. This is used in directory branching. + +## Content Addressing + +Bonsai uses Blake2b hashing to create a content-addressed Merkle DAG. Blake2b produces 256-bit (32-byte) hashes and provides cryptographic strength while being faster than SHA-256. + +### Hash Computation + +**Changeset Identifiers** +A Bonsai changeset is serialized (using Thrift compact protocol) and hashed with Blake2b. The resulting hash becomes the changeset identifier (`ChangesetId`). This identifier depends on: +- All metadata fields (author, dates, message, etc.) +- Parent changeset hashes +- All file changes (paths, content identifiers, types) +- Extra fields + +Any modification to the changeset produces a different identifier. + +**Content Identifiers** +File contents are hashed separately. Each file's content is hashed with Blake2b to produce a `ContentId`. Identical files across different commits or repositories produce the same content identifier, enabling deduplication. + +**Merkle DAG Properties** +The parent references in each changeset create a directed acyclic graph. The content-addressed nature ensures: +- Tampering detection (any change produces a different hash) +- Efficient comparison (identical hashes mean identical content) +- Deduplication (identical content stored once) + +The hash types are defined in `mononoke_types/src/hash.rs` and `mononoke_types/src/typed_hash.rs`. + +## File Content Storage + +File contents are stored separately from changesets in the blobstore. This separation provides several characteristics: + +**Content Blobs** +- Stored using their `ContentId` as the key +- Immutable once written +- Shared across all changesets that reference them + +**Chunking for Large Files** +File content can be stored in chunks for large files. This is managed by the filestore, which handles: +- Splitting large files into manageable chunks +- Reassembling chunks when retrieving files +- Optimizing transfer and storage of large binary files + +The `FileContents` type (defined in `mononoke_types/src/file_contents.rs`) can represent either: +- Direct bytes for small to medium files +- References to content chunks for large files + +## VCS-Agnostic Design + +Bonsai serves as an intermediate representation between different version control systems. The format is designed to capture the semantics of both Git and Mercurial while avoiding the implementation details of either. + +### Representation Capabilities + +**Git Compatibility** +Bonsai can represent all Git commit information: +- Author and committer as separate entities (matching Git's model) +- Committer date distinct from author date +- Git-specific headers preserved in `git_extra_headers +- Support for Git annotated tags + +**Mercurial Compatibility** +Bonsai can represent all Mercurial changeset information: +- Author (Mercurial typically has only author, not separate committer) +- Mercurial extra fields preserved in `hg_extra` +- Copy-from information for file renames + +**Simplified Structure** +Unlike both Git and Mercurial, Bonsai uses: +- Flat file change lists rather than nested tree structures +- Explicit copy-from information rather than heuristic rename detection +- Uniform handling of all file changes in a single structure + +### Conversion and Mapping + +Bonsai changesets are converted to and from VCS-specific formats: + +**Git Conversion** +- Git commits are imported by parsing the Git commit object and creating a corresponding Bonsai changeset +- Git trees are converted to the flat Bonsai file change list +- When serving Git clients, Bonsai changesets are converted back to Git commits and trees +- The `bonsai_git_mapping` table maintains bidirectional mappings between `ChangesetId` and Git commit SHA-1 + +**Mercurial Conversion** +- Mercurial changesets are imported by extracting metadata and file changes +- When serving Mercurial clients, derived data (Mercurial manifests and filenodes) provides the VCS-specific representation +- The `bonsai_hg_mapping` table maintains bidirectional mappings between `ChangesetId` and Mercurial changeset hash +- Original Mercurial changeset bytes are preserved as inherent data for exact reproduction + +These conversions allow Mononoke to serve Git and Mercurial clients from a single Bonsai backend while maintaining compatibility with each VCS. + +## Immutability and Storage + +Bonsai changesets are immutable. Once a changeset is created and stored: + +**Write-Once Semantics** +- Changesets are never modified after creation +- The content-addressed identifier ensures any change produces a different changeset +- Corrections require creating new changesets + +**Blobstore Storage** +Bonsai changesets are stored in the blobstore: +- Serialized using Thrift compact protocol +- Stored with a blobstore key derived from the `ChangesetId` +- Retrieved by key when needed +- Subject to blobstore caching and multiplexing layers + +**Metadata Database References** +While changesets are immutable, mutable state is maintained separately: +- The commit graph index (in the metadata database) tracks parent-child relationships for efficient queries +- VCS mapping tables connect Bonsai identifiers to Git and Mercurial identifiers +- Bookmarks (branch pointers) reference changeset identifiers but can be moved + +This separation of immutable content from mutable references allows efficient repository operations while maintaining data integrity. + +## Comparison with Git and Mercurial + +Understanding how Bonsai differs from Git and Mercurial clarifies the design choices: + +### Structure Differences + +**Git Trees vs. Bonsai File Changes** +Git represents repository state using nested tree objects. Each tree contains references to blobs (files) and subtrees (subdirectories), forming a recursive structure. Bonsai uses a flat list of file changes, where each entry specifies the complete path from the repository root. This eliminates the need to create and track intermediate tree objects. + +**Mercurial Manifests vs. Bonsai File Changes** +Mercurial uses manifest objects that, like Git trees, can be nested. Additionally, Mercurial maintains filelogs (per-file history) with linknodes connecting file revisions to changesets. Bonsai represents only what changed in each commit, with file history derived from the changeset graph. + +**Rename and Copy Tracking** +Git does not explicitly track renames or copies. Instead, Git detects renames heuristically when comparing trees. Mercurial explicitly records copy-from information in changeset metadata. Bonsai follows Mercurial's approach, explicitly storing copy-from information as part of tracked file changes. + +### Hash Algorithm + +**Git**: Uses SHA-1 (40 hex characters) for all object identifiers. Git is transitioning to SHA-256 (64 hex characters). + +**Mercurial**: Uses SHA-1 (40 hex characters) for changeset and manifest identifiers. + +**Bonsai**: Uses Blake2b (64 hex characters for 256-bit hashes). Blake2b provides cryptographic strength comparable to SHA-256 with better performance. + +### Metadata Handling + +**Git** distinguishes between author (who wrote the code) and committer (who committed it), each with their own timestamp. This supports workflows where commits are authored by one person and applied by another. + +**Mercurial** typically has only an author and date, though extra fields can store additional metadata. + +**Bonsai** includes both author and optional committer fields, accommodating Git's model while allowing Mercurial commits (which typically have no committer) to be represented naturally. + +## Type Definitions and Implementation + +The Bonsai data structures are defined in the `mononoke_types` crate, specifically: + +**Core Types** (`mononoke_types/src/bonsai_changeset.rs`) +- `BonsaiChangeset` - The immutable changeset structure +- `BonsaiChangesetMut` - The mutable builder used during changeset creation + +**File Changes** (`mononoke_types/src/file_change.rs`) +- `FileChange` - Enumeration of change types (tracked change, deletion, untracked change, untracked deletion) +- `TrackedFileChange` - A file addition or modification with copy-from information +- `BasicFileChange` - File change without copy-from tracking +- `FileType` - File type (regular, executable, symlink) + +**Content Addressing** (`mononoke_types/src/typed_hash.rs`) +- `ChangesetId` - Blake2b hash identifying a changeset +- `ContentId` - Blake2b hash identifying file contents + +**File Contents** (`mononoke_types/src/file_contents.rs`) +- `FileContents` - Either direct bytes or chunked content +- `ChunkedFileContents` - References to content chunks for large files + +**Paths** (`mononoke_types/src/path.rs`) +- `MPath` - A path within the repository (may include root) +- `NonRootMPath` - A path guaranteed not to be the root + +Additional types for dates, extra fields, and subtree changes are defined in their respective files within `mononoke_types/src/`. + +## Relationship to Derived Data + +While Bonsai changesets contain the core commit information, many repository operations require additional data structures. These are provided by derived data, which is computed from Bonsai changesets: + +**Manifests** +- `fsnodes` - Filesystem-like directory structure +- `unodes` - Full directory structure with file and directory history. Similar to Mercurial's manifests but with the "linknode problem" fixed. +- Skeleton manifests - Path-only directory structure for testing existence +- Git trees - Git-compatible tree objects + +**File Metadata** +- Filenodes - Per-file history information for Mercurial +- Blame - Line-by-line authorship attribution +- Fastlog - Optimized file history + +**VCS-Specific Formats** +- Git commits - Git commit objects for serving Git clients +- Git delta manifests - Precomputed Git deltas for efficient computation of pack files when serving clients +- Mercurial augmented manifests - Mecurial manifests with additional metadata for CASC + +These derived data types are described in detail in the Derived Data documentation. + +The separation between Bonsai (inherent data) and these structures (derived data) is a fundamental architectural decision. Bonsai provides the minimal representation needed for accepting pushes at high speed, while derived data provides the indexes and formats needed for efficient operations and VCS compatibility. + +## Working with Bonsai in Code + +When working with Mononoke code, Bonsai changesets are accessed through repository facets: + +**Reading Changesets** +The `repo_blobstore` facet provides access to the blobstore, allowing changesets to be loaded by their `ChangesetId`. The `Loadable` trait (implemented for `ChangesetId`) enables loading the corresponding `BonsaiChangeset` from storage. + +**Creating Changesets** +New Bonsai changesets are typically created using `BonsaiChangesetMut`, which provides a builder pattern for assembling the changeset components. Once complete, the mutable changeset is frozen into an immutable `BonsaiChangeset` and stored. + +**Commit Graph Traversal** +The `commit_graph` facet provides efficient parent-child traversal without loading full changesets. This enables operations like ancestry checking and graph walking without deserializing all changeset metadata. + +**VCS Mapping** +The `bonsai_git_mapping` and `bonsai_hg_mapping` facets provide bidirectional lookup between Bonsai changeset identifiers and Git or Mercurial commit hashes. These are used extensively when serving VCS clients. + +Repository operations compose these facets to implement higher-level functionality like pushrebase, cross-repository sync, and hooks. + +## Summary + +The Bonsai data model serves as Mononoke's canonical representation of version control data. Its design reflects several architectural principles: + +**VCS Independence** - Bonsai is not tied to Git or Mercurial implementation details, allowing it to serve as an intermediate representation. + +**Content Addressing** - Blake2b hashing creates a Merkle DAG ensuring data integrity and enabling efficient deduplication. + +**Minimal Representation** - Bonsai contains only the information necessary to represent commit history, with additional indexes provided by derived data. + +**Flat File Changes** - The flat list of file changes simplifies many operations compared to nested tree structures. + +**Immutability** - Once created, Bonsai changesets never change, enabling aggressive caching and simplifying consistency reasoning. + +**Explicit Semantics** - Copy-from information, file types, and metadata are explicitly represented rather than inferred. + +## Related Documentation + +- [Key Concepts](1.2-key-concepts.md) - Brief introduction to Bonsai and other core concepts +- [Architecture Overview](1.3-architecture-overview.md) - How Bonsai fits into the overall system +- [Derived Data](2.3-derived-data.md) - Data structures computed from Bonsai changesets +- [Repository Facets](2.2-repository-facets.md) - How to access Bonsai data through facets +- [Storage Architecture](2.4-storage-architecture.md) - How Bonsai changesets are stored diff --git a/eden/mononoke/docs/2.2-repository-facets.md b/eden/mononoke/docs/2.2-repository-facets.md new file mode 100644 index 0000000000000..79d7c4439c2f3 --- /dev/null +++ b/eden/mononoke/docs/2.2-repository-facets.md @@ -0,0 +1,345 @@ +# Repository Facets + +This document explains the facet pattern and how repositories are composed in Mononoke. Facets are trait-based components that provide specific repository capabilities. Understanding facets is essential for working with Mononoke, as they form the foundation of how code accesses repository functionality. + +## What Are Facets? + +A facet is a trait-based component that provides a specific capability for a repository. Rather than having a single repository class with all possible methods, Mononoke breaks repository functionality into discrete facets that can be composed together. Each facet encapsulates a single responsibility and may contain state that forms part of the repository. + +For example, `RepoIdentity` provides repository name and ID, `RepoBlobstore` provides access to immutable blob storage, and `CommitGraph` provides commit graph traversal operations. Functions declare their requirements explicitly by specifying which facets they need through trait bounds. + +Facets are defined in the `repo_attributes/` directory and are used throughout Mononoke's codebase as the standard way to access repository functionality. As explained in the [Architecture Overview](1.3-architecture-overview.md), facets form the composition layer between base components (storage, types, utilities) and higher-level features (pushrebase, cross-repo sync). + +## Defining Facets + +Facets are defined using the `#[facet::facet]` macro, which generates the necessary infrastructure for the facet pattern. Here's a simple example from `repo_attributes/repo_identity/src/lib.rs`: + +```rust +#[facet::facet] +#[derive(Clone, Hash, PartialEq, Eq)] +pub struct RepoIdentity { + id: RepositoryId, + name: String, +} + +impl RepoIdentity { + pub fn id(&self) -> RepositoryId { + self.id + } + + pub fn name(&self) -> &str { + &self.name + } +} +``` + +The `#[facet::facet]` macro generates: +- Accessor traits (e.g., `RepoIdentityRef`, `RepoIdentityArc`) for accessing the facet from generic containers +- Type aliases for Arc-wrapped facets (e.g., `ArcRepoIdentity`) +- Integration with the facet container system + +Facets can hold state (like configuration, cached data, or storage handles) or simply provide access to underlying systems. More complex facets like `CommitGraph` and `RepoBlobstore` wrap storage backends and provide domain-specific operations. + +## Using Facets in Code + +Functions declare their facet requirements using trait bounds. This makes dependencies explicit and allows the compiler to verify that the necessary capabilities are available. Here's an example: + +```rust +use commit_graph::CommitGraph; +use repo_blobstore::RepoBlobstore; +use repo_identity::RepoIdentity; + +async fn example_operation( + ctx: &CoreContext, + repo: &(impl CommitGraph + RepoBlobstore + RepoIdentity), + cs_id: ChangesetId, +) -> Result<()> { + // Access facets through generated accessor methods + let repo_name = repo.repo_identity().name(); + let blobstore = repo.repo_blobstore(); + let parents = repo.commit_graph() + .changeset_parents(ctx, cs_id) + .await?; + + // Function only has access to the three declared facets + Ok(()) +} +``` + +This pattern has several characteristics: + +**Explicit Dependencies** - The function signature declares exactly which repository capabilities are required. A function needing only identity and blobstore access cannot accidentally depend on commit graph operations. + +**Composability** - Different repository types can provide different combinations of facets. Test repositories can provide mock implementations of specific facets. + +**Compile-Time Verification** - If a function attempts to access a facet not declared in its trait bounds, the code will not compile. + +For functions requiring many facets, trait aliases can reduce verbosity: + +```rust +pub trait MyOperationRepo = CommitGraph + + RepoBlobstore + + RepoIdentity + + RepoDerivedData; + +async fn complex_operation( + ctx: &CoreContext, + repo: &impl MyOperationRepo, +) -> Result<()> { + // Function can use all four facets +} +``` + +## Available Facets + +Mononoke provides approximately 35 facets, organized into several functional categories. Each facet is implemented in its own directory under `repo_attributes/` or related top-level directories. + +### Identity and Configuration + +**RepoIdentity** (`repo_attributes/repo_identity/`) +- Provides repository ID and name +- Required by nearly all repository operations + +**RepoBookmarkAttrs** (`repo_attributes/repo_bookmark_attrs/`) +- Bookmark-specific configuration attributes +- Defines policies for bookmark operations + +### Storage Access + +**RepoBlobstore** (`repo_attributes/repo_blobstore/`) +- Access to the repository's immutable blobstore +- Wrapped with prefix and redaction layers specific to the repository + +**Filestore** (`repo_attributes/filestore/`) +- File content storage and retrieval +- Handles file chunking for large files + +**MutableBlobstore** (`repo_attributes/mutable_blobstore/`) +- Mutable blob operations +- Used for temporary or mutable repository data + +### Commit Graph and History + +**CommitGraph** (`repo_attributes/commit_graph/commit_graph/`) +- Efficient commit graph storage and traversal +- Provides ancestry queries, parent/child relationships, graph traversal + +**Phases** (`repo_attributes/phases/`) +- Commit phase tracking (draft, public) +- Important for Mercurial/Sapling semantics + +### Derived Data + +**RepoDerivedData** (`repo_attributes/repo_derived_data/`) +- Access to the derived data manager for this repository +- Coordinates derivation and fetching of all derived data types + +**RepoDerivationQueues** (`repo_attributes/repo_derivation_queues/`) +- Queues for coordinating remote derivation +- Manages work distribution across derivation workers + +### VCS Mappings + +These facets map between Bonsai (Mononoke's internal format) and external VCS identities: + +**BonsaiHgMapping** (`repo_attributes/bonsai_hg_mapping/`) +- Bonsai ↔ Mercurial changeset ID mapping +- Essential for Mercurial/Sapling client support + +**BonsaiGitMapping** (`repo_attributes/bonsai_git_mapping/`) +- Bonsai ↔ Git commit SHA mapping +- Required for Git protocol support + +**BonsaiGlobalrevMapping** (`repo_attributes/bonsai_globalrev_mapping/`) +- Bonsai ↔ GlobalRev (sequential integer) mapping +- Provides SVN-style sequential commit identifiers + +**BonsaiSvnrevMapping** (`repo_attributes/bonsai_svnrev_mapping/`) +- Bonsai ↔ SVN revision mapping +- Used for repositories imported from Subversion + +**BonsaiBlobMapping** (`repo_attributes/bonsai_blob_mapping/`) +- Maps bonsai changeset IDs to all the blobs that were introduced by that changeset +- Used for enumerating blobs when deleting changesets + +**BonsaiTagMapping** (`repo_attributes/bonsai_tag_mapping/`) +- Maps Git annotated tag objects + +### Bookmarks and References + +**Bookmarks** (`repo_attributes/bookmarks/bookmarks/`) +- Read and write access to bookmarks (branch pointers) + +**BookmarkUpdateLog** (within `repo_attributes/bookmarks/`) +- History of all bookmark movements of non-scratch bookmarks +- Used for synchronization and auditing + +**BookmarksCache** (within `repo_attributes/bookmarks/`) +- Cached bookmark information +- Reduces database load for frequently accessed bookmarks + +### Git-Specific Facets + +**GitSymbolicRefs** (`repo_attributes/git_symbolic_refs/`) +- Git symbolic references (like HEAD → refs/heads/main) +- Required for proper Git protocol support + +**GitRefContentMapping** (`repo_attributes/git_ref_content_mapping/`) +- Maps Git references to their content +- Handles Git tag objects and other ref types + +**GitSourceOfTruth** (`repo_attributes/git_source_of_truth/`) +- Tracks which system is the authoritative source for Git data +- Used during migration from MetaGit to Mononoke + +### File and Data Management + +**Filenodes** (`repo_attributes/filenodes/`) +- Legacy file history information for Mercurial +- Maps files to changesets that modified them (Mercurial linknodes) + +**Newfilenodes** (`repo_attributes/newfilenodes/`) +- Updated filenode implementation + +**MutableCounters** (`repo_attributes/mutable_counters/`) +- Integer counters for tracking operation progress +- Used by sync jobs and other background processes + +**MutableRenames** (`repo_attributes/mutable_renames/`) +- Allows dynamic modification of historical file rename information +- Used to fix up errors in file history + +### Operational Facets + +**RepoPermissionChecker** (`repo_attributes/repo_permission_checker/`) +- Permission and ACL verification +- Controls read and write access to repository data + +**HookManager** (`repo_attributes/hook_manager/`) +- Repository hook execution +- Enforces pre-land bookmark policies and validations + +**RepoCrossRepo** (`repo_attributes/repo_cross_repo/`) +- Cross-repository sync operations +- Enables repository-to-repository synchronization (megarepo) + +**RepoLock** (`repo_attributes/repo_lock/`) +- Repository-level locking + +**RepoSparseProfiles** (`repo_attributes/repo_sparse_profiles/`) +- Sparse checkout profile size tracking + +**RestrictedPaths** (`repo_attributes/restricted_paths/`) +- Path-level access restrictions + +### Metadata and Events + +**PushrebaseMutationMapping** (`repo_attributes/pushrebase_mutation_mapping/`) +- Tracks commit rewrites during pushrebase +- Maps original commits to their rewritten versions + +**DeletionLog** (`repo_attributes/deletion_log/`) +- Part of draft commit deletion + +**RepoMetadataCheckpoint** (`repo_attributes/repo_metadata_checkpoint/`) +- Checkpoint for the repo metadata logger + +**RepoEventPublisher** (`repo_attributes/repo_event_publisher/`) +- Manages subscriptions to repository events (bookmark updates) + +**SqlQueryConfig** (`repo_attributes/sql_query_config/`) +- Configuration for SQL query behavior +- Controls timeouts, retry policies, and other SQL parameters + +## Facet Construction and Composition + +### Repository Factory + +The `repo_factory/` directory contains the code responsible for constructing repositories from configuration. The factory: + +1. Reads repository configuration from the configuration system +2. Constructs storage backends (blobstore, SQL databases) +3. Creates each facet with its dependencies +4. Assembles facets into a facet container + +The resulting repository object implements all the accessor traits, allowing code to access any facet through the appropriate trait bound. + +### Test Repositories + +Test code uses `TestRepoFactory` to create repositories with test-appropriate backends. For example: + +```rust +let factory = TestRepoFactory::new()?; +let repo = factory + .with_name("test_repo") + .with_id(RepositoryId::new(1)) + .build() + .await?; +``` + +Test repositories typically use in-memory storage (`memblob`, SQLite) rather than production backends. Individual facets can be mocked or replaced for testing specific behaviors. + +## Relationship to Features + +As described in the [Architecture Overview](1.3-architecture-overview.md), features sit one layer above facets. Features are implemented as functions that combine multiple facets to provide high-level source control operations. Unlike facets, features do not hold state—they orchestrate operations across repository attributes. + +For example, the pushrebase feature (in `features/pushrebase/`) combines facets like `CommitGraph`, `Bookmarks`, `RepoBlobstore`, `HookManager`, and `RepoDerivedData` to implement server-side rebasing. The feature function declares its facet requirements through trait bounds and uses those facets to perform the operation. + +This separation allows features to be reused across different repository types and contexts. A pushrebase function can be called from the main server (via the API layer), from the land service (directly), or from the admin tool (for testing), all using the same implementation. + +## Facet Design Principles + +Several patterns are consistent across facet implementations: + +**Single Responsibility** - Each facet encapsulates one aspect of repository functionality. `RepoIdentity` provides identity, `CommitGraph` provides graph operations, and so on. + +**State Encapsulation** - Facets can hold state (configuration, storage handles, caches) but expose this state through well-defined methods rather than public fields. + +**Storage Abstraction** - Facets abstract over storage details. `CommitGraph` can use SQL storage, in-memory storage, or cached storage without changing its interface. + +**Async Operations** - Operations that perform I/O are async, allowing concurrent access and efficient resource usage. + +**Arc-Wrapped Types** - Facets are typically used through `Arc` (atomic reference counting) to enable shared ownership across async operations. The macro generates `Arc*` type aliases for convenience. + +## Finding Facet Implementations + +All repository attribute facets are located in `repo_attributes/`. Each facet has its own subdirectory containing: +- `src/` - Rust source code +- `BUCK` - Build definitions +- Optional: SQL storage implementation in a subdirectory (e.g., `sql_bookmarks/`) + +The [Navigating the Codebase](1.4-navigating-the-codebase.md) guide provides additional detail on finding and understanding facet implementations. + +## Patterns for Working with Facets + +When writing code that uses repositories: + +**Declare Minimal Requirements** - Only require the facets actually needed. A function that only reads repository identity should require `impl RepoIdentity`, not all possible facets. + +**Use Trait Aliases for Complex Requirements** - If many functions need the same set of facets, define a trait alias to reduce duplication. + +**Avoid Storing Facets Directly** - Use `Arc` wrappers when facets need to be stored. The generated `Arc*` type aliases make this straightforward. + +**Access Through Trait Methods** - Use the generated accessor methods (e.g., `repo.repo_identity()`) rather than attempting to downcast or access internals. + +When adding new facets: + +**Follow the Pattern** - Use `#[facet::facet]` macro and follow existing facet structure. + +**Place in repo_attributes** - New facets should be subdirectories of `repo_attributes/`. + +**Document the Interface** - Add rustdoc comments explaining what the facet provides. + +**Register with Factory** - Update `repo_factory/` to construct the new facet when creating repositories. + +## Next Steps + +This document covered the facet pattern and how repositories are composed from facets. To dive deeper: + +- **[Architecture Overview](1.3-architecture-overview.md)** - How facets fit into the overall system architecture +- **[Navigating the Codebase](1.4-navigating-the-codebase.md)** - Finding facet implementations in the directory structure +- **[Bonsai Data Model](2.1-bonsai-data-model.md)** - The core data model that facets operate on +- **[Derived Data](2.3-derived-data.md)** - How the `RepoDerivedData` facet manages derived data types +- **[Storage Architecture](2.4-storage-architecture.md)** - How `RepoBlobstore` and metadata storage facets work + +The facet pattern is used consistently throughout Mononoke. Understanding how to declare facet requirements and access facet functionality is fundamental to working with the codebase. diff --git a/eden/mononoke/docs/2.3-derived-data.md b/eden/mononoke/docs/2.3-derived-data.md new file mode 100644 index 0000000000000..cbb7a6988d282 --- /dev/null +++ b/eden/mononoke/docs/2.3-derived-data.md @@ -0,0 +1,361 @@ +# Derived Data + +This document explains Mononoke's derived data system—the framework for computing and caching indexes and alternative representations from the canonical Bonsai changesets and file content blobs. + +## What is Derived Data? + +Derived data is information computed from Bonsai changesets and file contents that can always be regenerated from the core data. While Bonsai changesets and file content blobs constitute the source of truth for repository history, many operations would be inefficient using only this minimal representation. Derived data provides indexes, directory structures, file metadata, and VCS-specific formats that enable performant read operations. + +Examples of derived data include: +- Directory manifests that allow traversing file hierarchies +- File history information for blame and log operations +- Git trees and commits for serving Git clients +- Mercurial filenodes and manifests for Mercurial protocol compatibility +- Case conflict detection data +- Deleted file tracking + +The distinguishing characteristic of derived data is that it can be computed asynchronously after a commit has been accepted, rather than requiring synchronous computation during the commit operation. + +## Separation from Core Data + +Mononoke separates repository data into two categories: + +**Core Data** - Information that must be written synchronously during commit operations: +- Bonsai changesets (parents, author, message, file changes) +- File content blobs +- VCS mappings (Bonsai ↔ Git/Mercurial identifiers) +- Commit graph index (parent-child relationships) + +**Derived Data** - Information computed from core data after the commit completes: +- Directory manifests (various types) +- File history and blame +- VCS-specific formats (Git trees, Mercurial manifests) +- Repository analysis data + +This separation allows Mononoke to maintain high commit throughput. When a commit is pushed, only core data is written before the operation completes and the next commit can proceed. Derived data is computed asynchronously by background workers or on-demand when needed for read operations. + +### Write Path and Read Path + +The write path accepts commits and stores core data: + +1. Validate incoming commit from client (Git or Sapling) +2. Convert to Bonsai format +3. Store Bonsai changeset and file contents in blobstore +4. Update VCS mapping in metadata database +5. Add to commit graph +6. Run hooks and update bookmark if being pushed to a public bookmark +8. Push completes + +Derived data computation is not on this critical path. The next commit can proceed immediately. + +The read path uses derived data to serve queries efficiently: + +1. Client requests data (file content, directory listing, blame) +2. Server determines which derived data type is needed +3. Check if derived data exists for the requested changeset +4. If not derived, compute it +5. Return result to client + +For repositories with high commit rates, this separation prevents derived data computation from becoming a bottleneck. Additional derived data types can be added without affecting write latency. + +## The Derived Data Framework + +Derived data types are implemented using a trait-based framework centered on the `BonsaiDerivable` trait. This framework handles dependency management, caching, batch derivation, and storage. + +### BonsaiDerivable Trait + +Each derived data type implements `BonsaiDerivable`, which defines: + +**Core Methods:** +- `derive_single` - Compute derived data for one changeset given its Bonsai representation and the derived data for its parents +- `store_mapping` - Store the derived data in a way that can be retrieved by changeset ID +- `fetch` - Retrieve previously computed derived data by changeset ID + +**Optimization Methods:** +- `derive_batch` - Compute derived data for multiple changesets efficiently (default: sequential derivation) +- `fetch_batch` - Retrieve derived data for multiple changesets efficiently (default: individual fetches) + +**Dependencies:** +- `Dependencies` type - Other derived data types that must be available before this type can be derived +- `PredecessorDependencies` type - Optional predecessor types that can be used for parallel backfilling + +The framework is located in `derived_data/` and `derived_data/manager/`. + +### Dependency Graph + +Derived data types can depend on other derived data types. These dependencies form a directed acyclic graph that the derivation framework respects. When deriving a changeset, all dependencies are derived first. + +Example dependencies: +- Blame depends on Unodes (file history manifests) +- Basename Suffix Skeleton Manifest V3 depends on Skeleton Manifest V2 +- Git Delta Manifest V2 depends on Git Commit +- Mercurial Augmented Manifest depends on Mercurial Changesets + +Dependencies are declared using the `Dependencies` associated type in the `BonsaiDerivable` implementation. The framework automatically ensures dependencies are satisfied before derivation proceeds. + +### Storage + +Derived data is stored in the blobstore using content-addressed keys. Each derived data type defines its own key format, typically incorporating the changeset ID and a type-specific prefix. + +Example key patterns: +- `derived_root_blame_v2.` - Blame data +- `derived_root_fsnode.` - Fsnode manifest root + +Mappings from changeset IDs to derived data are also stored in the metadata database for some types, providing fast lookups without requiring blobstore access. + +## Derivation Processes + +Derived data can be computed in several ways depending on operational needs. + +### On-Demand Derivation + +When a client requests data that requires a specific derived data type, and that data has not yet been derived for the requested changeset, derivation occurs on-demand. The server derives the data, stores it, and returns the result. + +On-demand derivation follows the dependency graph: if type A depends on type B, and neither is derived, type B is derived first, then type A. + +### Batch Derivation + +For bulk operations or backfilling, derived data can be computed in batches. The `derive_batch` method allows implementations to optimize derivation across multiple changesets by: +- Batching database queries +- Amortizing common computations +- Parallelizing independent derivations + +Batch derivation is used by the derived data service and bulk derivation tools. + +### Backfilling + +Backfilling is the process of deriving data for all changesets in a repository, typically after introducing a new derived data type or upgrading to a new version. Backfilling can be performed: + +**Sequentially** - Derive changesets in topological order from repository roots to heads. Each changeset has its parents already derived. + +**In Parallel with Slicing** - Divide the commit graph into slices, derive data for slice boundaries using predecessor dependencies, then derive each slice in parallel. + +**Using Predecessor Optimization** - Some types support `derive_from_predecessor`, which can compute derived data using a different derived data type without requiring parent data. This enables parallel backfilling by deriving "anchor points" first. + +Backfilling is performed using the admin CLI or bulk derivation tools in `derived_data/bulk_derivation/`. + +### Incremental Derivation + +After backfilling or initial derivation, new commits are derived incrementally as they arrive. This is typically handled by: +- On-demand derivation when data is requested +- Background workers deriving data for recent commits +- The derived data service coordinating derivation across workers + +The warm bookmark cache mechanism ensures that derived data for public bookmarks (like `master`) is kept up-to-date. + +## Remote Derivation Service + +The Remote Derivation Service (`derived_data/remote/`) provides centralized, asynchronous derivation. Instead of each Mononoke server deriving data locally, derivation requests are enqueued and processed by a pool of derivation workers. + +**Architecture:** +- **Service** - Thrift API server that accepts derivation requests and enqueues them +- **Workers** - Background processes that poll the queue and perform derivation +- **Queue** - Dependency-aware queue that ensures derived data dependencies are satisfied (implemented in `repo_attributes/repo_derivation_queues/`) + +**API:** +- `derive` - Request asynchronous derivation, returns a token +- `poll` - Check status of a derivation request using the token + +**Benefits:** +- Deduplication of derivation work (multiple requests for the same changeset coalesce) +- Horizontal scaling of derivation capacity by adding workers +- Reduced memory pressure on frontend servers +- Centralized control and monitoring of derivation + +The service is defined in `derived_data/remote/if/derived_data_service.thrift` and implemented in `facebook/derived_data_service/`. + +## Derived Data Types + +Mononoke includes a variety of derived data types serving different purposes. These are organized into several categories. + +### Manifest Types + +Manifests represent directory structures in different formats. Manifests can be tree-based or flat, and can also be sharded or unsharded. Unsharded manifests suffer from performance issues as the number of files in a directory increases, while sharded manifests can scale to large directories. + +**Fsnodes** (`derived_data/fsnodes/`) +- File and directory manifest using a tree structure +- Stores file metadata (size, content hash, file type) +- Used for directory traversal and file lookups +- Not sharded + +**Unodes** (`derived_data/unodes/`) +- Manifest with file history linkage +- Tracks file and directory parentage for history traversal +- Allows following file history without traversing all commits +- Not sharded + +**Skeleton Manifest** (`derived_data/skeleton_manifest/`) +- Directory structure without file contents +- Used for path-based queries +- Supports case conflict detection (legacy) +- Not sharded + +**Skeleton Manifest V2** (`derived_data/skeleton_manifest_v2/`) +- Updated skeleton manifest using Sharded Map V2 +- Sharded +- Does not support case conflict handling (moved to separate type) + +**Basename Suffix Skeleton Manifest V3** (`derived_data/basename_suffix_skeleton_manifest_v3/`) +- Indexed by file basename suffix and path prefix +- Enables efficient file search operations of the form `prefix/**/*.suffix`. +- Depends on Skeleton Manifest V2 +- Uses Sharded Map V2 for improved performance with large directories +- Flat and sharded + +**Case Conflict Skeleton Manifest** (`derived_data/case_conflict_skeleton_manifest/`) +- Tracks case conflicts in paths +- Separated from Skeleton Manifest V2 +- Flat and sharded + +**Content Manifest** (`derived_data/content_manifest_derivation/`) +- Alternative to Fsnodes using Sharded Map V2, still in development +- Content-addressed manifest representation +- Will also support Git LFS pointers in the future +- Sharded + +**Deleted Manifest V2** (`derived_data/deleted_manifest/`) +- Tracks deleted files and directories +- Enables efficient "was this path ever deleted?" queries +- Sharded using the legacy Sharded Map V1 + +### File Metadata Types + +**Filenodes** (`derived_data/filenodes_derivation/`) +- Mercurial-compatible file history +- Maps file paths to their history in Mercurial format +- Required for Mercurial protocol compatibility + +**Fastlog** (`derived_data/fastlog/`) +- Efficient file history lookup +- Pre-computed file history to avoid graph traversal + +**Blame V2** (`derived_data/blame/`) +- Line-by-line attribution of file contents +- Depends on Unodes for file history + +**Inferred Copy From** (`derived_data/inferred_copy_from/`) +- Copy/move detection using heuristics +- Similar to Git's copy detection + +### Git-Specific Types + +**Git Commit** (`git/git_types/`) +- Git commit objects derived from Bonsai +- Includes Git trees representing directory structure +- Enables serving Git protocol + +**Git Delta Manifest V2** (`git/git_types/`) +- Optimized representation for Git packfile generation +- Depends on Git Commit +- Uses Sharded Map V2 + +**Git Delta Manifest V3** (`git/git_types/`) +- Next generation delta manifest +- Further optimizations for packfile generation + +### Mercurial-Specific Types + +**Mercurial Changeset** (`derived_data/mercurial_derivation/`) +- Mercurial changeset derived from Bonsai +- Required for Mercurial protocol compatibility +- Includes Mercurial manifests representing directory structure + +**Mercurial Augmented Manifest** (`derived_data/mercurial_derivation/`) +- Content-addressed Mercurial manifest +- Alternative to traditional Mercurial manifests necessary for integration with CASC +- Includes additional metadata + +### Utility Types + +**Changeset Info** (`derived_data/changeset_info/`) +- Cached changeset metadata (author, message, parents) +- Enables fast metadata queries without deserializing full Bonsai changeset + +**Test Manifest** (`derived_data/test_manifest/`) +- Manifest type for testing the derived data framework +- Not used in production + +**Test Sharded Manifest** (`derived_data/test_sharded_manifest/`) +- Sharded manifest for testing +- Validates Sharded Map implementation + +## Sharded Maps + +Several manifest types use Sharded Maps, a specialized data structure for storing large mappings across multiple blobstore blobs. Sharded Maps allow loading subsets of a manifest without fetching the entire structure. + +**Sharded Map V2** - Improved implementation with: +- Trie-based structure with path compression +- Terminal nodes for small subtrees (stored inline) +- Bounded node sizes even for very large directories +- Efficient partial loading for path lookups + +Sharded Maps are used by Skeleton Manifest V2, Basename Suffix Skeleton Manifest V3, Content Manifest, and Git Delta Manifest V2. + +## Versioning and Migration + +Derived data types can be versioned, allowing the data model to evolve while maintaining backward compatibility. + +**Versioning Strategy:** +- New versions are implemented as separate derived data types (e.g., Blame → Blame V2) +- Old versions continue to exist until no longer needed +- Repositories can be migrated by backfilling the new version +- Configuration controls which version is active per repository + +**Adding a New Version:** +1. Implement new type with `BonsaiDerivable` +2. Define storage format and keys +3. Add to repository configuration +4. Backfill for existing changesets +5. Update clients to use new version +6. Deprecate and remove old version when migration is complete + +This allows derived data improvements without downtime or risky migrations. + +Existing types can be rederived using a mapping key prefix, defined in `DerivedDataTypesConfig::mapping_key_prefixes`. This prefix is added to all mapping keys before the changeset id, and separates them into a different namespace so that rederivation can be performed independently. + +## Adding New Derived Data Types + +To add a new derived data type: + +1. **Define the Type** - Create a struct representing the derived data +2. **Implement BonsaiDerivable** - Define how to derive from Bonsai and parents +3. **Declare Dependencies** - Specify any required derived data types +4. **Define Storage** - Implement `store_mapping` and `fetch` methods +5. **Register Type** - Add to the `DerivableType` enum in `mononoke_types` +6. **Configure** - Add to repository configuration to enable +7. **Test** - Write unit and integration tests +8. **Backfill** - Derive for existing changesets + +**Example locations:** +- Derived data implementations: `derived_data/*/` +- Framework: `derived_data/manager/` +- Type registration: `mononoke_types/src/derivable_type.rs` +- Configuration: `metaconfig/types/` + +The `BonsaiDerivable` trait documentation in `derived_data/src/lib.rs` provides detailed usage information. + +## Performance Considerations + +The derived data system is designed for performance at scale: + +**Batch Derivation** - Deriving multiple changesets together reduces database round-trips and allows amortization of common work. + +**Dependency Management** - The framework ensures dependencies are derived in the correct order, minimizing redundant derivation. + +**Remote Derivation** - Centralizing derivation reduces duplicate work across servers and enables horizontal scaling. + +**Caching** - Derived data is stored in the blobstore and benefits from multi-level caching (cachelib, memcache). + +**Incremental Computation** - Most derived data types can compute new values based on parent values, avoiding full recomputation. + +**Selective Derivation** - Not all derived data types need to be derived for all changesets. Configuration controls which types are active. + +## Related Documentation + +* [Architecture Overview](1.3-architecture-overview.md) - System context and write/read path separation +* [Key Concepts](1.2-key-concepts.md) - Brief derived data introduction +* [Bonsai Data Model](2.1-bonsai-data-model.md) - Core data that derived data builds upon +* [Repository Facets](2.2-repository-facets.md) - RepoDerivedData facet +* [Storage Architecture](2.4-storage-architecture.md) - How derived data is stored + +Component-specific documentation for individual derived data types lives in the respective directories under `derived_data/`. diff --git a/eden/mononoke/docs/2.4-storage-architecture.md b/eden/mononoke/docs/2.4-storage-architecture.md new file mode 100644 index 0000000000000..2eb40bf8341ec --- /dev/null +++ b/eden/mononoke/docs/2.4-storage-architecture.md @@ -0,0 +1,391 @@ +# Storage Architecture + +This document explains Mononoke's storage architecture, including the blobstore system, metadata database, caching strategy, and how these components work together to provide scalable repository storage. + +## Storage Model Overview + +Mononoke has two main data stores: + +**Immutable Blobstore** - Content-addressed key-value storage for all immutable data including Bonsai changesets, file contents, and derived data. Once written, blobs are never modified or deleted (except through explicit redaction processes). + +**Metadata Database** - SQL database (MySQL in production, SQLite for development) storing repository state that changes over time, such as bookmarks, VCS mappings, and the commit graph index. + +This separation allows different optimization strategies for each type of data. The blobstore can use aggressive caching and replication since data is immutable, while the database handles transactional updates for mutable state. + +In addition to these main stores, Mononoke also stores some things in other data stores: + +**Mutable Blobstore** - Storage for pre-computed data like the pre-loaded commit graph. This data is periodically recomputed and overwritten in the mutable blobstore. + +**Ephemeral Blobstore** - Garbage collectable blobstore used for ephmeral objects like snapshots. The ephemeral blobstore groups blobs into "bubbles" that expire together. + +**Zelos Derivation Queue** - A graph-aware queue that is used to co-ordinate derivation of derived data. + +## Blobstore Architecture + +The blobstore is Mononoke's primary storage layer for repository content. All blobs are identified by unique keys and are immutable after creation. + +### What Lives in the Blobstore + +The blobstore stores several categories of data: + +**Core Repository Data:** +- **Bonsai changesets** - Commit metadata including author, message, timestamps, parents, and file changes +- **File content blobs** - Actual file contents, chunked for large files +- **File content metadata** - Metadata about file contents + +**Derived Data:** +- **Manifests** - Directory structures (fsnodes, unodes, skeleton manifests) +- **File metadata** - blame annotations, fastlog entries +- **Mercurial-specific data** - Augmented Mercurial manifests +- **Git-specific data** - Git delta manifests +- **Utility data** - Changeset info, deleted manifest, case conflict detection + +**VCS-Specific Formats:** +- **Mercurial artifacts** - Revlog-compatible file nodes and manifests +- **Git artifacts** - Git commit and tree objects + +Most blobs use content-addressed keys based on Blake2b hashing, which enables deduplication and integrity verification. Some blobs are keyed using the hash of a corresponding item, for example blame data is keyed using unode hashes. + +### Storage Backends + +Mononoke supports multiple blobstore backend implementations: + +**Production Backends:** + +- **Manifoldblob** (`facebook/manifoldblob/`) - Primary production backend using Manifold, Meta's internal distributed storage service. This is the standard deployment option for Meta's infrastructure. + +- **SQLblob** (`blobstore/sqlblob/`) - SQL database backend supporting both MySQL (production) and SQLite (development). Stores blobs as rows in a table with the key as the primary key and the blob bytes as a column. + +- **S3blob** (`blobstore/s3blob/`) - Amazon S3 backend for deployments using AWS infrastructure. + +**Development and Testing Backends:** + +- **Fileblob** (`blobstore/fileblob/`) - Filesystem-based storage where each blob is a file. Used for local development and testing. + +- **Memblob** (`blobstore/memblob/`) - In-memory storage with no persistence. Used exclusively for unit tests. + +Backend implementations share a common `Blobstore` trait, making them interchangeable. The choice of backend is a deployment decision that does not affect higher-level code. + +### The Decorator Pattern + +Mononoke uses the decorator pattern to compose blobstore functionality. Rather than implementing features directly in backend storage implementations, decorators wrap blobstores to add specific capabilities. This allows flexible configuration of the storage stack. + +**Core Decorators:** + +- **Prefixblob** (`blobstore/prefixblob/`) - Adds a prefix to all keys, typically used to namespace keys by repository. This allows multiple repositories to share a single blobstore backend without key collisions. + +- **Cacheblob** (`blobstore/cacheblob/`) - Implements multi-level caching using both cachelib (in-process cache) and memcache (shared cache across servers). Caching is discussed in detail in the Caching Strategy section below. + +- **Multiplexedblob** (`blobstore/multiplexedblob/`) - Writes blobs to multiple underlying blobstores simultaneously and reads from any available backend. This provides redundancy and availability, discussed further in the Multiplexing section. + +- **Packblob** (`blobstore/packblob/`) - Packs multiple related blobs together and applies compression. This reduces storage costs and is detailed in the Packblob section below. + +**Operational Decorators:** + +- **Redactedblobstore** (`blobstore/redactedblobstore/`) - Enforces content redaction policies, preventing access to blobs that have been marked as redacted. Used for removing sensitive content from repositories. + +- **Samplingblob** (`blobstore/samplingblob/`) - Samples operations for observability, logging a configurable fraction of all blobstore operations for monitoring and debugging. + +- **Throttledblob** (`blobstore/throttledblob/`) - Applies rate limiting to blobstore operations, controlling load on downstream storage systems. + +- **Logblob** (`blobstore/logblob/`) - Logs all blobstore operations for debugging and auditing purposes. + +- **Readonlyblob** (`blobstore/readonlyblob/`) - Wraps a blobstore to make it read-only, preventing writes. Used when operating in read-only modes. + +**Testing Decorators:** + +- **Chaosblob** (`blobstore/chaosblob/`) - Randomly fails operations to test error handling and resilience. + +- **Delayblob** (`blobstore/delayblob/`) - Adds artificial latency to operations to test timeout handling and performance under slow storage. + +**Specialized Storage:** + +- **Ephemeral blobstore** (`blobstore/ephemeral_blobstore/`) - Temporary storage for snapshots and draft commits that have not been made permanent. Uses a separate SQL table with time-based expiration. + +- **Virtually sharded blobstore** (`blobstore/virtually_sharded_blobstore/`) - Caching blobstore that groups blobstore keys into virtual shards to allow deduplication of in-flight requests. + +Decorators can be stacked in different orders to achieve different behaviors. The factory pattern in `blobstore/factory/` constructs the appropriate stack based on configuration. + +### Example Blobstore Stack + +A typical production blobstore stack composes multiple decorators: + +``` +Application (repo attributes, features, etc.) + ↓ +Prefixblob (adds repo-specific key prefix) + ↓ +Cacheblob (memcache + cachelib caching) + ↓ +Multiplexedblob (write to multiple backends) + ↓ +Packblob (compression) + ↓ +Backend storage (sqlblob, manifoldblob, or s3blob) +``` + +In this stack: + +1. **Application code** accesses the blobstore through repository facets +2. **Prefixblob** namespaces keys by repository ID (e.g., `repo123.bonsai.abc123...`) +3. **Cacheblob** checks cachelib and memcache before querying lower layers +4. **Multiplexedblob** coordinates writes to multiple backends and reads from any available backend +5. **Packblob** packs related blobs and applies compression +6. **Backend storage** persists blobs to the underlying storage system + +Additional decorators can be inserted at various points in the stack. For example, redactedblobstore typically sits near the top to prevent redacted content from being accessed, while samplingblob might sit near the bottom to observe actual backend operations. + +### Multiplexing for Availability + +Mononoke's storage must remain available even when individual storage systems have issues. A particular concern is cyclic dependencies: if Mononoke uses a storage system, and that storage system's infrastructure uses Mononoke for source control, then a storage outage could prevent deploying fixes to that storage system. + +To address this, Mononoke writes blobs to multiple independent storage backends simultaneously. The multiplexedblob decorator implements a write-all, read-any strategy: + +**Write Operations:** +- Blobs are written to all configured backends +- The write succeeds if a quorum of backends acknowledges the write +- If some backends fail, the write is logged to a write-ahead log (WAL) + +**Read Operations:** +- Reads are submitted to all back-ends by default +- The first backend to successfully return the blob satisfies the read +- If one backend is unavailable, reads succeed from other backends + +**Consistency Maintenance:** +- The blobstore healer job (`jobs/blobstore_healer/`) periodically reads the WAL +- For each incomplete write, the healer copies blobs from backends that have the data to backends that are missing it +- Once all backends have the blob, the WAL entry is deleted + +**Backend Independence:** +- Backends are chosen to be as independent as possible +- For example, multiple Manifold buckets might use separate ZippyDB tiers +- One backend might be sqlblob while another is manifoldblob +- This reduces the probability that all backends fail simultaneously + +The multiplexedblob WAL is stored in the metadata database (tables defined in `blobstore_sync_queue/schemas/`). + +### Packblob: Compressed Storage + +Packblob (`blobstore/packblob/`) reduces storage costs by packing multiple blobs together and applying compression. + +**Packing Strategy:** + +Packblob can pack multiple key-value pairs into a single underlying blob. Related blobs (such as different versions of the same file) can be packed together, improving compression ratios and reducing the number of objects stored in the underlying backend. + +**Compression:** + +Packblob uses Zstd compression with delta compression. One blob serves as a dictionary, and other blobs in the pack are compressed using deltas against that dictionary. This is effective for Mononoke data where related blobs (such as different versions of the same file) share significant content. + +**Storage Envelope:** + +All blobs stored through packblob are wrapped in a Thrift-based storage envelope that indicates whether the blob is stored independently or as part of a pack. This discriminator allows reads to determine the storage layout without requiring external metadata. + +**Position in Stack:** + +Packblob sits between the underlying storage backend and the multiplexedblob decorator. This allows different backends in a multiplex to use different packing strategies. The typical stack is: + +``` +multiplexedblob + ↓ +packblob + ↓ +backend storage (sqlblob, manifoldblob, etc.) +``` + +**Cache Interaction:** + +Packblob can cache both the packed form (compressed blobs) and the unpacked form (individual blobs). The packed form might be cached close to the storage backend to reduce decompression overhead, while the unpacked form is cached at higher levels for application access. + +See `blobstore/packblob/README.md` for detailed design documentation. + +## Metadata Database + +While the blobstore holds immutable content, the metadata database stores mutable repository state and indexes that change as the repository evolves. + +### What Lives in the Metadata Database + +The metadata database contains several categories of mutable data: + +**Bookmarks and References:** + +- **Bookmarks table** - Current positions of all branches (bookmarks), mapping bookmark names to Bonsai changeset IDs. Schema defined in `repo_attributes/bookmarks/dbbookmarks/schemas/`. + +- **Bookmarks update log** - Audit trail of all bookmark movements, recording the changeset a bookmark moved from, the changeset it moved to, the reason for the move, and a timestamp. This provides a complete history of branch updates. + +**VCS Mappings:** + +- **Bonsai-Mercurial mapping** (`bonsai_hg_mapping`) - Bidirectional mapping between Bonsai changeset IDs and Mercurial changeset hashes. Schema in `repo_attributes/bonsai_hg_mapping/schemas/`. + +- **Bonsai-Git mapping** (`bonsai_git_mapping`) - Bidirectional mapping between Bonsai changeset IDs and Git commit SHA-1 hashes. Schema in `repo_attributes/bonsai_git_mapping/schemas/`. + +- **Bonsai-Globalrev mapping** (`bonsai_globalrev_mapping`) - Mapping between Bonsai changeset IDs and sequential integer globalrevs used in some workflows. Schema in `repo_attributes/bonsai_globalrev_mapping/schemas/`. + +- **Bonsai-SVN revision mapping** (`bonsai_svnrev_mapping`) - Mapping for repositories imported from Subversion. Schema in `repo_attributes/bonsai_svnrev_mapping/schemas/`. + +**Commit Graph Index:** + +- **Commit graph tables** - Index of parent-child relationships in the commit graph with precomputed skip tree pointers for efficient ancestry queries. Schema in `repo_attributes/commit_graph/sql_commit_graph_storage/schemas/`. This includes generation numbers, merge ancestors, and skip tree structure. + +**Repository State:** + +- **Phases** - Tracks whether commits are in draft or public phase. Schema in `repo_attributes/phases/sqlphases/schemas/`. + +- **Mutable counters** - Integer counters used for cross-repository sync and other operational tasks. Schema in `repo_attributes/mutable_counters/schemas/`. + +- **Deletion log** - Records content deletions for auditing. Schema in `repo_attributes/deletion_log/schemas/`. + +- **Repository locks** - Allows locking of the repository for administrative actions. Schema in `repo_attributes/repo_lock/schemas/`. + +**Additional Mappings and Metadata:** + +- **Git references** - Git symbolic refs and ref content mappings. Schemas in `repo_attributes/git_symbolic_refs/schemas/` and `repo_attributes/git_ref_content_mapping/schemas/`. + +- **Pushrebase mutation mapping** - Tracks how commits were rewritten during pushrebase. Schema in `repo_attributes/pushrebase_mutation_mapping/schemas/`. + +- **Cross-repo sync mappings** - Maps commits between synchronized repositories. Schema in `features/commit_rewriting/synced_commit_mapping/schemas/`. + +- **Filenodes** - Mercurial filenode data for protocol compatibility. Schema in `repo_attributes/newfilenodes/schemas/`. + +All SQL schemas support both MySQL (production) and SQLite (development), with schema files located in component-specific `schemas/` directories. + +### Blobstore vs. Metadata Database Decision + +The choice of where to store data follows these principles: + +**Store in the blobstore when:** +- Data is immutable after creation +- Data is content-addressed +- Data is large or varies significantly in size +- Data benefits from caching + +**Store in the metadata database when:** +- Data is mutable and requires transactional updates +- Data is small and fixed-size +- Data requires indexed queries or range scans +- Data represents relationships between objects + +For example, a Bonsai changeset is immutable and content-addressed, so it lives in the blobstore. A bookmark is mutable and requires atomic updates, so it lives in the metadata database. The mapping between a Bonsai changeset ID and its Mercurial hash is immutable once established but requires indexed lookups in both directions, so it lives in the metadata database. + +## Caching Strategy + +Mononoke employs multi-level caching to reduce load on storage backends and improve performance. + +### Cache Levels + +**Level 1: In-Process Cache (Cachelib)** + +Each Mononoke server process maintains an in-memory cache using Cachelib, Meta's internal caching library. This cache is process-local and not shared between servers. Data is serialized using bincode for efficient memory representation. + +- Cache size is configurable +- Eviction uses LRU or other configured policies +- Cache keys are the same as blobstore keys +- Only positive results are cached (successful reads) + +**Level 2: Shared Cache (Memcache)** + +A shared memcache cluster provides caching across all Mononoke servers in a region. Data is serialized using custom formats defined by the `MemcacheEntity` trait (often using Thrift Compact Protocol for complex types). + +- Shared across all server instances +- Larger capacity than per-process cache +- Network overhead for cache hits +- TTLs control how long data is cached + +**Cache Hierarchy:** + +When fetching a blob: + +1. Check cachelib (process-local cache) +2. If miss, check memcache (shared cache) +3. If miss, fetch from blobstore backend +4. On backend fetch, populate both memcache and cachelib +5. On memcache hit, populate cachelib + +This hierarchy means that frequently accessed data can be served from in-process memory, moderately accessed data comes from the shared cache, and infrequently accessed data requires backend access. + +Some backend storage implementations (notably manifold) have their own cache, which is below the Mononoke caches. + +### Cache Configuration + +Cache behavior can be controlled through command-line flags: + +- `--cache-mode=local-only` - Use only cachelib, skip memcache +- `--cache-mode=cache-disabled` - Disable all caching +- Additional flags control cache sizes and policies + +Different cache modes are useful for testing, debugging, and scenarios where caching behavior needs to be controlled. + +### Warm Bookmark Cache + +Because of the asynchronous nature of derived data, if a client requests the value of a bookmark immediately after a bookmark has been updated and receives a newly landed commit, any subsequent requests for that commit will be delayed while data for that commit is derived. To prevent this, Mononoke maintains a cache of bookmarks that are "warm", which means derived data has been derived for the commit that the bookmark points to. This has the effect of delaying visibility of bookmark updates until after data derivation has happened. This delay is typically small (a few seconds) and has the same characteristics as the eventual consistency property of distributed systems. + +Mononoke services can maintain the warm bookmark cache locally. Alternatively, the bookmark service microservice can maintain this cache and serve bookmark-related queries to front-end services. This prevents duplication of effort of cache maintenance across multiple hosts and reduces the load on the metadata database. + +### Cache Warming: Microwave + +Microwave (`microwave/`) is a cache warming system that preloads caches before they are needed. When a new server starts up or when caches are cold, microwave can: + +- Preload derived data for recent commits on important bookmarks +- Warm blobstore caches with frequently accessed blobs +- Prepare caches before traffic is directed to a server + +This reduces the "cold start" problem where a server with empty caches would perform poorly until caches are populated through normal access patterns. + +Microwave was particularly important for the Mercurial wireproto implementation. It is less important now, and may be removed in the future. + +### Cacheblob Implementation + +The cacheblob decorator (`blobstore/cacheblob/`) implements the multi-level caching strategy. It wraps an underlying blobstore and intercepts get and put operations: + +**On get:** +1. Hash the key to create cache keys for both cachelib and memcache +2. Check cachelib +3. If cachelib miss, check memcache +4. If memcache hit, populate cachelib and return +5. If memcache miss, fetch from underlying blobstore +6. Populate both cachelib and memcache before returning + +**On put:** +- Write to underlying blobstore +- Optionally populate caches (configuration-dependent) + +The caching implementation in `common/caching_ext/` provides utilities for implementing caches with proper serialization, batching, and chunking of requests. + +The virtually sharded blobstore is a variant of cacheblob that shards the blobstore keyspace into multiple shards to allow deduplication of simultaneous requests through the cache. + +## Storage Characteristics + +Mononoke's storage architecture has several characteristics that affect how the system operates: + +**Immutability Enables Aggressive Caching** - Since most blobstore data never changes, caches can retain data indefinitely without worrying about invalidation. TTLs are used primarily to manage cache capacity rather than consistency. + +**Multiplexing Trades Write Latency for Availability** - Writing to multiple backends increases write latency but provides redundancy. This trade-off is acceptable because write operations are less frequent than reads, and high availability is critical. + +**Separation of Storage Types Enables Optimization** - The blobstore and metadata database can be optimized differently. The blobstore uses content-addressed keys and eventual consistency, while the metadata database uses transactional updates and indexed queries. + +**Decorator Composition Increases Flexibility** - The decorator pattern allows different repositories to use different storage configurations without changing application code. A repository storing large binary files might use packblob differently than a repository with mostly source code. + +**Caching Reduces Backend Load** - Multi-level caching reduces load on storage backends by several orders of magnitude for frequently accessed data. The majority of reads are served from cachelib without hitting network storage. + +**Storage is External to Servers** - Servers are stateless and can be added or removed without data migration. All persistent state lives in external storage systems. + +## Related Components + +Several other Mononoke components interact closely with storage: + +**Blobstore Healer** (`jobs/blobstore_healer/`) - Background job that ensures all blobs exist in all multiplexed backends. Reads the multiplexedblob WAL and repairs missing blobs. + +**Walker** (`jobs/walker/`) - Graph traversal tool that validates data integrity by walking the commit graph and verifying that all referenced blobs exist and are readable. See `jobs/walker/src/README.md` for details. + +**Filestore** (`filestore/`) - Higher-level abstraction over the blobstore for storing file contents. Handles chunking of large files, content deduplication, and metadata storage. + +**Blobstore Factory** (`blobstore/factory/`) - Constructs blobstore stacks based on configuration, assembling the appropriate decorators and backends. + +## Related Documentation + +- **[Architecture Overview](1.3-architecture-overview.md)** - How storage fits into the overall system architecture +- **[Key Concepts](1.2-key-concepts.md)** - Introduction to blobstore and metadata database concepts +- **[Bonsai Data Model](2.1-bonsai-data-model.md)** - What data is stored in the blobstore +- **[Derived Data](2.3-derived-data.md)** - How derived data is stored +- **[Walker and Validation](6.2-walker-and-validation.md)** - Storage validation and scrubbing + +Component-specific storage details are documented in the respective component directories. diff --git a/eden/mononoke/docs/3.1-servers-and-services.md b/eden/mononoke/docs/3.1-servers-and-services.md new file mode 100644 index 0000000000000..b203d35b313bc --- /dev/null +++ b/eden/mononoke/docs/3.1-servers-and-services.md @@ -0,0 +1,239 @@ +# Servers and Services + +This document describes Mononoke's servers and microservices, their responsibilities, and how they interact with the repository layer. + +## Overview + +Mononoke is deployed as a distributed service architecture with multiple server types. Frontend services handle external client requests, while internal microservices manage expensive operations and provide coordination. All services are stateless, with persistent state stored in external storage (blobstore and metadata database). + +The separation of frontend services from microservices allows independent scaling and workload isolation. Frontend services can scale horizontally to handle client load, while microservices can be scaled based on the characteristics of specific operations (CPU-intensive derivation, serialized landing operations, etc.). + +## Frontend Services + +Frontend services implement various protocols to serve external clients. Each service is stateless and can be deployed as multiple instances behind a load balancer. + +### Mononoke Server (SLAPI/EdenAPI) + +**Location**: `server/` +**Binary**: `fbcode//eden/mononoke:mononoke` +**Clients**: Sapling CLI, EdenFS + +The Mononoke server is the primary frontend service for Sapling and EdenFS clients. It implements the EdenAPI (also called SLAPI - Sapling Remote API) protocol over HTTP, providing source control operations for Sapling-based workflows. + +The server handles clone, pull, and push operations from Sapling clients, as well as on-demand file fetches from EdenFS instances. Request handling is implemented in the `repo_listener/` subdirectory, which manages TCP connections, TLS termination, and routing to protocol handlers. The actual protocol implementation resides in `edenapi_service/`, which translates HTTP requests into operations on the repository layer. + +The server supports repository sharding, allowing repositories to be distributed across multiple server instances based on external shard assignment. Cache warmup is performed at startup for repositories assigned to each instance, preloading derived data and warming caches for optimal performance. + +### SCS Server (Source Control Service) + +**Location**: `scs/scs_server/` +**Binary**: `fbcode//eden/mononoke/scs/scs_server:scs_server` +**Clients**: SCMQuery, internal tools, automation + +The Source Control Service provides a Thrift API for programmatic repository access. While the Mononoke server implements VCS protocols (Git, Sapling), SCS exposes a high-level, language-agnostic API for tools and services that need to query or manipulate repositories. + +The Thrift interface (`scs/if/source_control.thrift`) defines approximately 80 methods for repository operations: commit lookups, history traversal, blame annotation, file content retrieval, directory listings, and diff generation. Write operations include commit creation and bookmark manipulation. The service supports multiple commit identity schemes (Bonsai, Git, Mercurial, Globalrev, SVN), translating between formats transparently using VCS mapping facets. + +SCS is used by internal services for repository introspection, code review systems for commit information, and automation tools for repository management. The service handles authentication via mTLS and performs authorization checks through the repository permission checker facet. + +### Git Server + +**Location**: `git_server/` +**Binary**: `fbcode//eden/mononoke/git_server:git_server` +**Clients**: Standard Git clients + +The Git server implements the Git protocol over HTTP, allowing standard Git clients to interact with Mononoke repositories. It handles Git clone, fetch, and push operations using Git's pack protocol and object formats. + +The server translates between Git's data model (commits, trees, blobs) and Mononoke's Bonsai format using the Git-specific facets and derived data types. Git commits are mapped to Bonsai changesets via `bonsai_git_mapping`, while Git trees are derived from Bonsai file changes. Push operations create Bonsai changesets from incoming Git commits and can land them using pushrebase if invoked from the git pushrebase client. + +The server supports Git's smart HTTP protocol, including upload-pack (fetch) and receive-pack (push) operations. It implements packfile generation for efficient data transfer and handles Git references (branches and tags) through symbolic refs facets. + +### LFS Server + +**Location**: `lfs_server/` +**Binary**: `fbcode//eden/mononoke/lfs_server:lfs_server` +**Clients**: Git clients, Sapling clients, EdenFS + +The LFS server implements the Git LFS (Large File Storage) protocol over HTTP. It handles upload and download of large files referenced by pointer files in Git and Sapling repositories. + +The server stores LFS objects in the blobstore using the filestore facet, which provides chunking for large files and content-addressed storage. Upload operations accept file content via HTTP POST and return object IDs (content hashes). Download operations serve file content by object ID, streaming data to avoid memory pressure for large files. + +The server can operate standalone or with an upstream LFS server for federation. Batch API endpoints allow clients to request upload or download URLs for multiple objects in a single request, reducing round trips during operations involving many large files. + +## Internal Microservices + +Internal microservices handle specialized operations that are offloaded from frontend servers. These services provide coordination, serialization, and dedicated scaling for specific workloads. + +### Land Service + +**Location**: `servers/land_service/` +**Interface**: `servers/land_service/if/land_service.thrift` +**Clients**: Frontend servers (Mononoke, SCS), automation + +The Land Service provides serialized commit landing to public bookmarks. It ensures that commits are landed atomically via pushrebase without race conditions from concurrent push operations. + +When multiple clients attempt to push to the same bookmark simultaneously, frontend servers forward requests to the Land Service rather than performing pushrebase locally. The service queues landing requests per bookmark and processes them serially, preventing wasted work from failed pushrebases where servers compete to move the same bookmark. + +The service executes the full pushrebase workflow: running pre-commit hooks, rebasing commits onto the current bookmark position, and moving the bookmark atomically. It returns the mapping of original commit IDs to rebased commit IDs, along with information about conflicts or hook failures. + +### Derived Data Service + +**Location**: `derived_data/remote/` (Thrift interface), `facebook/derived_data_service/` (server implementation) +**Interface**: `derived_data/remote/if/derived_data_service.thrift` +**Clients**: Frontend servers, derivation workers + +The Derived Data Service coordinates asynchronous derivation of derived data types. Rather than each server deriving data independently, this service deduplicates derivation requests and manages a pool of derivation workers. + +Frontend servers submit derivation requests to the service, which returns a token for polling completion status. Requests are placed in a dependency-aware queue that ensures derived data types are computed in the correct order (e.g., blame requires unodes, which require fsnodes). Background workers poll the queue and execute derivation using the derived data manager. + +This architecture reduces redundant computation (data derived once is available to all servers), provides centralized control over derivation resources, and allows horizontal scaling of derivation capacity by adding workers. + +### Additional Services + +**Bookmark Service** (Facebook-internal: `facebook/bookmark_service/`): Maintains a warm cache of public bookmarks with all derived data types computed, used by frontend servers to quickly serve queries for important branches. + +**Diff Service** (Facebook-internal: `facebook/diff_service/`): Provides diff computation as a service, offloading expensive diff generation from frontend servers. + +**Load Limiter Service** (Facebook-internal: `facebook/load_limiter/`): Implements rate limiting and load shedding across multiple frontend instances, coordinating QPS limits. + +These services follow the same pattern: Thrift interfaces, stateless operation, and integration with the repository layer through facets. + +## Using the Repository Layer + +All servers and services are built using the same internal architecture described in [Architecture Overview](1.3-architecture-overview.md). They use the `cmdlib/mononoke_app/` framework for initialization and access repositories through facets. + +### Repository Access via Facets + +Servers obtain repository capabilities through facets rather than accessing a monolithic repository object. A server declares its requirements via trait bounds, specifying which facets it needs: + +```rust +async fn handle_request(repo: &impl RepoBlobstore + Bookmarks + RepoDerivedData) { + // Access storage + let blobstore = repo.repo_blobstore(); + + // Query bookmarks + let bookmark = repo.bookmarks().get(...).await?; + + // Access derived data + let derived_data = repo.repo_derived_data(); +} +``` + +This pattern makes dependencies explicit and allows testing with minimal facet implementations rather than full repository instances. + +### Feature Integration + +Servers use features to implement high-level operations. Features are stateless functions that compose multiple facets. For example: + +**Pushrebase** (in `features/pushrebase/`): The Mononoke server and SCS server both use pushrebase when landing commits. The pushrebase feature requires several facets: bookmarks (to read and update target bookmark), commit graph (for ancestry queries), derived data (to compute manifests), and hook manager (to run validation). + +**Cross-repo sync** (in `features/cross_repo_sync/`): Used by servers handling cross-repository operations, this feature composes VCS mapping facets, commit transformation logic, and blobstore access. + +By using features, servers avoid reimplementing complex operations and benefit from shared, well-tested code. + +### API Layer Usage + +Frontend servers typically use the `mononoke_api/` layer, which provides high-level abstractions over facets. This layer exposes `Repo`, `Changeset`, and `File` objects that encapsulate common operations: + +```rust +let repo = mononoke_api.repo(...)?; +let changeset = repo.changeset(commit_id).await?; +let file = changeset.path_with_content(path).await?; +``` + +The API layer hides internal representation details (which manifest type, which commit graph implementation) and provides a stable interface as implementations evolve. + +## Communication Protocols + +Mononoke servers use HTTP and Thrift for communication. + +### HTTP-Based Protocols + +**EdenAPI (SLAPI)**: The Mononoke server implements EdenAPI over HTTP/2. Requests and responses use CBOR (Compact Binary Object Representation) encoding for efficient serialization. The protocol is designed around streaming and incremental operations - rather than transferring entire bundles, clients request specific files, trees, and commits as needed. + +The protocol supports content negotiation, compression, and multiplexing multiple requests over a single connection. Streaming responses allow serving large results without buffering entire datasets in memory. + +**Git Protocol**: The Git server uses Git's smart HTTP protocol, implementing the upload-pack and receive-pack services. Requests use Git's pkt-line format, and responses include packfiles containing Git objects (commits, trees, blobs). + +**LFS Protocol**: The LFS server follows the Git LFS specification, providing JSON-based batch API endpoints and binary upload/download endpoints. Upload URLs are generated by the batch endpoint and used by clients to PUT file content. Download URLs similarly provide GET access to stored objects. + +### Thrift-Based Services + +**SCS**: Uses Thrift RPC over HTTP with mTLS for authentication. The interface supports both synchronous methods (simple queries) and streaming methods (for operations returning large result sets). + +**Land Service**: Provides a single Thrift method (`land_changesets`) that accepts commit stacks and returns pushrebase outcomes or error details (conflicts, hook rejections). + +**Derived Data Service**: Asynchronous Thrift API where clients submit derivation requests and poll for completion. The service returns tokens that encode request state for later status queries. + +### Stateless Design + +All servers are stateless. They do not maintain per-client session state or cache data specific to individual requests. State resides in: + +- **Blobstore**: Immutable data (commits, files, derived data) +- **Metadata database**: Mutable state (bookmarks, VCS mappings, commit graph index) +- **Shared caches**: Memcache and cachelib for performance + +This design allows horizontal scaling - adding more server instances increases capacity without coordination overhead. Load balancers can route requests to any instance, and failed instances can be replaced without state migration. + +## Configuration and Sharding + +### Repository Sharding + +Mononoke supports two sharding modes: + +**Shallow sharding**: All repositories are loaded at startup. A load balancer routes requests for specific repositories to designated server instances. This mode is suitable for smaller deployments with tens or hundreds of repositories. + +**Deep sharding**: Repositories are loaded on-demand based on external shard assignment. A shard manager (external to Mononoke) determines which repositories each server instance should handle. Servers dynamically add and remove repositories as assignments change. This mode scales to thousands of repositories. + +Sharding configuration is specified in repository configs and command-line arguments (`--sharded-service-name`). Each service type can use different sharding strategies. + +### Server Configuration + +Servers share common configuration through `cmdlib/mononoke_app/`: + +- Repository configs (in `metaconfig/`) +- TLS certificates for mTLS +- Scuba logging +- Caching settings (cachelib, memcache) +- Monitoring and observability + +Service-specific configuration includes: + +- **Mononoke server**: Listen address, ALPN protocol negotiation, cache warmup settings +- **SCS server**: Thrift worker pool sizing, queue configuration, async request settings +- **Git server**: Upstream LFS server URL, rate limiting +- **LFS server**: Self URLs, upstream server, max upload size + +Configuration is loaded at startup, with some settings (rate limits, feature flags) supporting dynamic updates via cached config handles. + +## Monitoring and Observability + +All servers export metrics and logs through standard infrastructure: + +**Metrics**: ODS counters and histograms track QPS, latency (P50, P90, P99), error rates, and resource usage per endpoint/method. + +**Logging**: Scuba logging captures request details: repository, operation type, client identity, duration, outcome. Sampling rates can be configured per service. + +**Tracing**: Distributed tracing propagates request context through service calls, allowing end-to-end latency analysis across frontend servers and microservices. + +**Health checks**: Servers expose health endpoints and ready flags used by load balancers and orchestration systems. + +Monitoring infrastructure is initialized by the `mononoke_app` framework and integrated into each server's request handling middleware. + +## Summary + +Mononoke's service architecture separates concerns between frontend protocol servers and internal microservices. Frontend servers (Mononoke, SCS, Git, LFS) handle external clients and implement various protocols, while microservices (Land, Derived Data) manage expensive operations and provide coordination. + +All services use the same internal architecture: stateless operation, facet-based repository access, and shared library code. This consistency reduces maintenance burden and allows new services to be built by composing existing facets and features. + +The stateless design enables horizontal scaling and operational flexibility - capacity can be added by deploying more instances, and servers can be updated or replaced without state migration. Protocol diversity (HTTP, Thrift) supports different client types while maintaining a single canonical repository backend. + +For operational details about specific servers, refer to component directories: `server/`, `scs/`, `git_server/`, `lfs_server/`, `servers/land_service/`, and `derived_data/remote/`. + +## Related Documentation + +- [Architecture Overview](1.3-architecture-overview.md) - System and code architecture +- [Repository Facets](2.2-repository-facets.md) - How repositories are composed +- [Derived Data](2.3-derived-data.md) - Asynchronous data computation +- [Git Support](5.1-git-support.md) - Git integration details +- [Mercurial/Sapling Support](5.2-mercurial-sapling-support.md) - Sapling protocol details diff --git a/eden/mononoke/docs/3.2-jobs-and-background-workers.md b/eden/mononoke/docs/3.2-jobs-and-background-workers.md new file mode 100644 index 0000000000000..062b557db4925 --- /dev/null +++ b/eden/mononoke/docs/3.2-jobs-and-background-workers.md @@ -0,0 +1,219 @@ +# Jobs and Background Workers + +This document describes the background jobs and maintenance tasks that operate on Mononoke repositories. These processes run independently of user-facing servers and handle asynchronous operations, validation, and maintenance work. + +## Overview + +Mononoke's architecture separates user-facing operations from background maintenance. While servers handle client requests synchronously, background jobs perform work that does not need to block user operations. This separation allows the system to optimize the critical path for user requests while delegating expensive or periodic operations to dedicated workers. + +Background jobs fall into several categories: + +- **Validation and integrity** - Verify data consistency and storage durability +- **Asynchronous derivation** - Compute derived data off the critical path +- **Synchronization** - Replicate data across repositories or storage systems +- **Metrics and monitoring** - Collect repository statistics + +These jobs are typically long-running processes that continuously monitor for work, process batches of data, and report status via metrics and logging. + +## Walker + +**Location**: `jobs/walker/` + +The walker traverses the Mononoke commit graph and validates data integrity. It defines a graph schema that represents Mononoke's data model as nodes connected by edges, then uses bounded traversal to walk the graph efficiently. + +### Capabilities + +The walker supports several subcommands: + +**Scrub** - Verifies storage durability in multiplexed blobstores. When Mononoke uses multiple blobstore backends (for redundancy), the scrub operation checks that each component blobstore contains data for every key. If a blob is missing from one backend but present in another, the scrub can repair the inconsistency. This allows Mononoke to operate on a single component store if necessary. + +**Validate** - Checks data validity by traversing the graph and verifying references. This can detect issues like dangling references, missing data, or inconsistent metadata that might indicate bugs in derivation or write logic. + +**Corpus** - Extracts a subset of repository data based on sampling criteria. The corpus is written to disk in a structured directory layout, allowing offline analysis or experimentation with compression techniques. Sampling can be based on path patterns or hash values. + +**Compression Benefit** - Analyzes potential compression ratios by measuring the size of blobs before and after individual compression. This helps evaluate storage optimization strategies. + +### Implementation + +The walker represents Mononoke data as a directed graph where nodes correspond to Bonsai changesets, file contents, manifests, and other data types. Edges represent relationships like parent commits, file references, or manifest entries. The graph is discovered dynamically during traversal. + +Visit tracking prevents re-visiting nodes, which is necessary because the graph contains cycles (for example, filenode-to-changeset backlinks). The walker uses concurrent hash maps to track visited nodes while keeping memory usage bounded. + +Sampling support allows the walker to operate on subsets of large repositories. Sampling can be based on node hash or repository path, enabling operations to be split into batches that can be processed independently. + +**Documentation**: See `jobs/walker/src/README.md` for detailed information about the walker's architecture, memory management, and usage patterns. + +## Blobstore Healer + +**Location**: `jobs/blobstore_healer/` + +The blobstore healer maintains storage durability when using multiplexed blobstores. Mononoke writes to multiple independent storage backends simultaneously to avoid dependencies on any single storage system. The healer monitors a write-ahead log (WAL) queue and repairs any inconsistencies. + +### Operation + +The healer continuously processes entries from the blobstore synchronization queue. For each entry, it verifies that all component blobstores in the multiplex configuration contain the expected blob. If a blob is missing from one backend, the healer copies it from another backend that has the data. + +This addresses several scenarios: +- Temporary unavailability of a component blobstore during writes +- Blobs stored to only a subset of component stores due to configuration changes +- Corruption or data loss in a single backend + +The healer operates in batches, processing queue entries within configured age and concurrency limits. It can be run in dry-run mode to report issues without making changes, or in drain-only mode to clear processed entries without healing. + +### Configuration + +The healer accepts parameters for: +- Queue batch size and entry age thresholds +- Concurrency limits (number of blobs and total bytes) +- Shard ranges for parallel operation across multiple healer instances +- Storage configuration identifying which multiplexed blobstore to heal + +## Derived Data Workers + +**Location**: `derived_data/remote/` + +Derived data is computed asynchronously after commits are accepted. The remote derivation service coordinates this work across multiple worker processes, enabling horizontal scaling of derivation workload. + +### Remote Derivation Service + +The derived data service is a Thrift-based microservice that accepts derivation requests and delegates work to worker processes. Clients (typically Mononoke servers) send requests to derive specific data types for specific changesets. The service tracks derivation progress and deduplicates requests for the same data. + +This architecture allows derivation to scale independently from the main servers. During periods of high commit rate, additional derivation workers can be added to handle the load without affecting the write path. + +### Derivation Process + +When a commit is pushed, Mononoke stores the Bonsai changeset and file contents but does not compute derived data immediately. Instead, derivation happens asynchronously: + +1. A derivation request is queued or sent to the remote service +2. Workers fetch the Bonsai changeset and its dependencies +3. The worker computes the derived data (manifests, filenodes, etc.) +4. The derived data is stored in the blobstore +5. Metadata tables are updated to mark the data as derived + +Different derived data types have dependency relationships. For example, blame derivation depends on unodes, which must be derived first. The derivation framework manages these dependencies automatically. + +Derivation can be triggered on-demand (when a client requests data that hasn't been derived) or via batch jobs that derive data for all recent commits. + +## Cross-Repository Sync (Backsyncer) + +**Location**: `features/commit_rewriting/backsyncer/` + +The backsyncer synchronizes commits between different repositories based on defined mapping configurations. This supports workflows where a large repository is split into smaller repositories, or where commits need to be mirrored with path or content transformations. + +### Sync Process + +The backsyncer monitors the bookmark update log in a source repository. When bookmarks move (typically due to new commits), the backsyncer: + +1. Reads bookmark update log entries +2. Fetches the new commits from the source repository +3. Applies configured transformations (path rewriting, file filtering, etc.) +4. Creates equivalent commits in the target repository +5. Updates bookmarks in the target repository + +Transformations are defined in repository configuration and can include: +- Path prefix changes (moving all files under a different directory) +- File inclusion/exclusion patterns +- Commit message rewriting +- Author/timestamp preservation + +The backsyncer maintains mappings between source and target commits, allowing it to establish parent relationships correctly even when commit histories diverge. + +### Modes of Operation + +The backsyncer can run in different modes: +- Continuous tailing of the bookmark update log +- One-time sync of specific commits or bookmark ranges +- Sharded execution across multiple worker processes + +## CAS Sync + +**Location**: `jobs/cas_sync/` + +The CAS (Content Addressed Storage) sync job uploads commits and file contents to external content-addressed storage systems. This provides an additional layer of data redundancy and enables integration with other systems that consume repository data. + +### Operation + +The CAS sync job monitors the bookmark update log and processes new commits. For each commit, it: + +1. Fetches the commit metadata and file contents +2. Uploads data to the CAS backend +3. Records progress in mutable counters +4. Logs sync status to Scuba + +The job handles retries for failed uploads and can be configured to sync specific bookmarks. It maintains a counter of the most recently synced bookmark update log entry, allowing it to resume from the correct position after restarts. + +## Modern Sync + +**Location**: `jobs/modern_sync/` + +The modern sync job synchronizes repository data to remote endpoints using the EdenAPI protocol. This enables replication of repository data to other Mononoke instances or compatible systems. + +### Operation + +Similar to CAS sync, modern sync processes bookmark update log entries. It groups entries into batches and syncs the associated commits and derived data to the target system. The job supports: + +- Batching of bookmark updates for efficiency +- Configurable sync targets via TLS and proxy settings +- Progress tracking via mutable counters +- Optional ODS (operational data store) metrics + +The job can operate in sharded mode, with multiple instances processing different repositories. + +## Statistics Collector + +**Location**: `jobs/statistics_collector/` + +The statistics collector computes repository metrics such as file counts, total file size, and line counts. These metrics are tracked over time and logged to Scuba for monitoring and analysis. + +### Operation + +The statistics collector monitors a bookmark (typically the main branch) and recomputes statistics when new commits appear. For each commit, it: + +1. Fetches the manifest from the commit +2. Iterates over all files in the repository +3. Computes statistics (file count, total size, line counts) +4. Logs results to Scuba with the commit timestamp + +For subsequent commits, the collector computes incremental changes by examining the diff between manifests, which is more efficient than recalculating statistics from scratch. + +The collector can operate in two modes: +- Continuous monitoring of a bookmark +- One-time computation for a set of commits from a file + +Statistics are computed only for regular files below a size threshold (to avoid processing very large files). Redacted content is handled by reporting zero lines for redacted files. + +## Job Scheduling and Orchestration + +Background jobs are deployed as independent processes or services. Scheduling and lifecycle management depends on the deployment environment: + +**Sharded Execution** - Jobs that process multiple repositories (like the statistics collector or backsyncer) can use the sharded executor framework. This integrates with Meta's ShardManager to automatically distribute repositories across worker instances and handle instance failures. + +**Continuous Operation** - Jobs like the blobstore healer run continuously, polling their input queues and processing batches. They include sleep periods when no work is available to avoid excessive database load. + +**Triggered Jobs** - Some jobs (like walker scrub operations) are triggered periodically via external scheduling systems rather than running continuously. + +## Monitoring and Alerting + +Background jobs report their status through several mechanisms: + +**Metrics** - Jobs publish ODS counters and timeseries metrics for processed items, queue depths, error rates, and processing latency. These metrics power operational dashboards and alerting. + +**Scuba Logging** - Detailed logs of individual operations (synced commits, healed blobs, derived changesets) are written to Scuba tables. This enables debugging and historical analysis. + +**Health Checks** - Jobs integrate with Mononoke's monitoring framework to report liveness and readiness. This allows orchestration systems to detect and restart unhealthy instances. + +**Error Reporting** - Errors are logged with context (repository, changeset ID, operation type) to facilitate investigation. + +Jobs typically include command-line flags for dry-run mode, iteration limits, and custom logging, allowing operators to test changes safely and control resource usage. + +## Relationship to Servers and Tools + +Background jobs occupy a distinct layer in Mononoke's architecture: + +- **Servers** handle synchronous client requests and must respond quickly +- **Jobs** handle asynchronous operations and can take longer to process batches +- **Tools** are invoked on-demand by operators for administrative tasks + +Jobs use the same library code as servers and tools. They access repositories through the same facet-based abstractions and use the same storage backends. The difference is in their execution model: jobs are long-running processes that continuously look for work, while servers are stateless request handlers and tools are one-shot commands. + +This separation allows Mononoke to optimize each layer independently. Servers minimize latency by avoiding expensive operations. Jobs maximize throughput by processing batches. Tools provide flexibility for operators to perform manual interventions when needed. diff --git a/eden/mononoke/docs/3.3-tools-and-utilities.md b/eden/mononoke/docs/3.3-tools-and-utilities.md new file mode 100644 index 0000000000000..cdd30af883190 --- /dev/null +++ b/eden/mononoke/docs/3.3-tools-and-utilities.md @@ -0,0 +1,358 @@ +# Tools and Utilities + +This document describes the command-line tools and utilities available for operating and maintaining Mononoke. These tools provide operational capabilities for administrators, debugging interfaces for developers, and utilities for data operations. + +## Overview + +Mononoke provides a collection of command-line tools for various operational tasks. These tools are distinct from servers (which handle client requests) and jobs (which run as long-lived background processes). Tools are invoked on-demand by operators or administrators to perform specific tasks such as importing repositories, verifying data integrity, or performing maintenance operations. + +The primary administrative interface is the `admin` tool, which provides a comprehensive set of subcommands for most operational tasks. Additional specialized tools handle specific operations like repository import, data verification, and storage maintenance. + +## Tool vs Job vs Server + +Understanding the distinction between these three types of applications is important for navigating the codebase and choosing the appropriate component for a task: + +**Tools** (in `tools/`) +- Command-line utilities invoked on-demand +- Execute a specific task and exit +- Examples: `admin`, `blobimport`, `aliasverify` +- Used by operators and administrators for operational tasks +- Can run in production or development environments + +**Jobs** (in `jobs/`) +- Long-running background processes +- Continuously perform maintenance or monitoring tasks +- Examples: `walker`, `blobstore_healer`, `statistics_collector` +- Typically scheduled and monitored as services +- Described in detail in [Jobs and Background Workers](3.2-jobs-and-background-workers.md) + +**Servers** (various top-level directories) +- Handle client requests over network protocols +- Run continuously as services +- Examples: Mononoke server, Git server, SCS server +- Described in detail in [Servers and Services](3.1-servers-and-services.md) + +This document focuses on tools—the command-line utilities used for operational and administrative tasks. + +## Administrative Tools + +### Admin CLI + +**Location**: `tools/admin/` + +**Binary**: `fbcode//eden/mononoke/tools/admin:admin` + +The admin tool is the primary command-line interface for Mononoke operators. It provides a unified interface for inspecting and modifying repository state, managing derived data, performing data operations, and debugging issues. + +The admin tool is organized into subcommands, each handling a specific aspect of repository administration. The tool includes over 140 command files implementing various operations. + +**Major subcommand categories:** + +**Repository Operations** +- `bookmarks` - Bookmark (branch) management and inspection +- `changelog` - Changeset log operations +- `commit` - Commit inspection and operations +- `fetch` - Fetch files and changesets + +**Derived Data** +- `derived-data` - Derived data management, derivation, and inspection +- `derivation-queue` - Derivation queue operations +- `blame` - Blame data inspection +- `case-conflict` - Case conflict detection + +**Storage Operations** +- `blobstore` - Blobstore inspection and operations +- `raw-blobstore` - Direct blobstore access +- `ephemeral-store` - Ephemeral storage management +- `filestore` - File storage operations + +**Commit Graph** +- `commit-graph` - Commit graph operations and inspection + +**Cross-Repository Operations** +- `cross-repo` - Cross-repository sync operations +- `cross-repo-config` - Cross-repo configuration management +- `megarepo` - Megarepo (monorepo) operations + +**Git-Specific Operations** +- `git-bundle` - Git bundle operations +- `git-symref` - Git symbolic reference management +- `git-content-ref` - Git content reference operations +- `git-objects` - Git object inspection +- `git-cgdm-updater` - Git commit graph delta manifest updates +- `git-cgdm-components` - Git CGDM component inspection + +**Other Operations** +- `mutable-counters` - Mutable counter management +- `mutable-renames` - Mutable rename tracking +- `phases` - Commit phase management +- `redaction` - Content redaction operations +- `locking` - Repository locking +- `async-requests` - Asynchronous request management +- `cas-store` - Content-addressed storage operations +- `repo-info` - Repository information +- `list-repos` - List configured repositories +- `convert` - Conversion utilities +- `slow-bookmark-mover` - Controlled bookmark movement + +The admin tool is the recommended interface for most operational tasks. Operators should use this tool rather than directly manipulating storage or database state. + +### Testtool + +**Location**: `tools/testtool/` + +**Binary**: `fbcode//eden/mononoke/tools/testtool:testtool` + +The testtool provides utilities for testing and debugging Mononoke in non-production environments. This tool cannot be run against production repositories—it validates the environment and refuses to execute if production configuration is detected. + +The testtool is used during development and testing to perform operations that would be inappropriate for production environments. It provides debugging capabilities and test utilities not available in the admin tool. + +## Import and Export Tools + +### Blobimport + +**Location**: `tools/blobimport/` + +**Binary**: `fbcode//eden/mononoke/tools/blobimport:blobimport` + +Blobimport imports Mercurial repositories into Mononoke's blobstore format. The tool reads Mercurial revlog data and converts it to Bonsai changesets and associated data structures. + +The import process: +1. Reads Mercurial changelog and manifest data +2. Converts changesets to Bonsai format +3. Stores file contents in the blobstore +4. Creates VCS mappings (Bonsai ↔ Mercurial) +5. Optionally imports bookmarks +6. Optionally derives specified data types + +Blobimport supports various options for controlling the import process, including parent order fixes, bookmark policies, and derived data derivation. + +### Gitimport + +**Location**: `git/gitimport/` + +**Binary**: `fbcode//eden/mononoke/git/gitimport:gitimport` + +Gitimport imports Git repositories into Mononoke. The tool reads Git objects and converts them to Bonsai changesets while preserving Git-specific metadata. + +The import process: +1. Reads Git objects from a Git repository +2. Converts commits to Bonsai format +3. Creates Git-specific derived data (Git trees, commits) +4. Establishes VCS mappings (Bonsai ↔ Git) +5. Handles Git references and symbolic refs +6. Optionally processes Git LFS objects + +Gitimport supports incremental imports, allowing repositories to be updated with new commits after the initial import. + +### Repo Import + +**Location**: `tools/repo_import/` + +**Binary**: `fbcode//eden/mononoke/tools/repo_import:repo_import` + +The repo_import tool handles repository import with additional capabilities beyond basic blobimport or gitimport. It supports: +- Cross-repository sync during import +- Commit transformation and rewriting +- Backsyncing operations +- Large repository imports with specialized handling + +This tool is used for complex import scenarios where simple blobimport or gitimport is insufficient. + +### Streaming Clone + +**Location**: `tools/streaming_clone/` + +**Binary**: `fbcode//eden/mononoke/tools/streaming_clone:streaming_clone` + +The streaming_clone tool manages streaming clone chunks for Mercurial clients. Streaming clone allows clients to quickly obtain a repository's changelog by downloading pre-generated chunks rather than fetching individual changesets. + +The tool provides subcommands to: +- Create new streaming clone chunks from Mercurial revlog data +- Update existing streaming clone data +- Manage chunk storage in the blobstore + +Streaming clone chunks are stored in Mononoke's mutable blobstore and served to compatible Mercurial clients. + +## Verification Tools + +### Aliasverify + +**Location**: `tools/aliasverify/` + +**Binary**: `fbcode//eden/mononoke/tools/aliasverify:aliasverify` + +Aliasverify validates content-addressed aliases in the blobstore. File contents in Mononoke are identified by multiple hash algorithms (SHA-1, SHA-256, Blake2, Blake3, Git SHA-1). The tool verifies that: +- Alias blobs exist for file contents +- Aliases correctly point to the canonical content +- Hash computations are accurate + +The tool can operate in several modes: +- Verify aliases for specific changesets +- Process ranges of changesets +- Run as a sharded service for large-scale verification + +Aliasverify is used to detect and repair missing or corrupted alias data. + +### Bonsai Verify + +**Location**: `tools/bonsai_verify/` + +**Binary**: `fbcode//eden/mononoke/tools/bonsai_verify:bonsai_verify` + +Bonsai_verify validates the consistency between Bonsai changesets and derived Mercurial data. The tool: +- Computes Mercurial manifests from Bonsai changesets +- Compares computed manifests against stored manifests +- Validates the Bonsai ↔ Mercurial conversion process +- Checks manifest structure and file metadata + +The tool can verify specific changesets, process ranges, or validate entire repository histories. It reports discrepancies between Bonsai and Mercurial representations, which indicate data corruption or conversion errors. + +### Check Git Working Copy + +**Location**: `tools/check_git_wc/` + +**Binary**: `fbcode//eden/mononoke/tools/check_git_wc:check_git_wc` + +This tool validates that a Bonsai changeset produces the same working copy as a Git commit. It: +- Computes the working copy from a Bonsai changeset +- Compares against a Git repository's working copy at a specific commit +- Reports differences in file contents or tree structure +- Optionally handles Git LFS pointer files + +The tool is used to verify the correctness of Git ↔ Bonsai conversions and ensure that Mononoke produces working copies identical to native Git. + +## Maintenance Tools + +### Packer + +**Location**: `tools/packer/` + +**Binary**: `fbcode//eden/mononoke/tools/packer:packer` + +The packer tool manages packblob operations. Packblob is a blobstore decorator that compresses multiple small blobs into larger packed blobs, reducing storage overhead and improving retrieval efficiency. + +The tool accepts a set of blob keys and: +- Fetches the blobs from storage +- Compresses them using zstd with a specified compression level +- Packs them into larger multi-blob containers +- Uploads the packed blobs to the blobstore + +The packer supports parallel packing operations and can run in dry-run mode to estimate compression benefits without modifying storage. See `blobstore/packblob/README.md` for details on the packblob format. + +### SQLblob Garbage Collection + +**Location**: `tools/sqlblob_gc/` + +**Binary**: `fbcode//eden/mononoke/tools/sqlblob_gc:sqlblob_gc` + +The sqlblob_gc tool performs garbage collection on SQLblob storage. SQLblob stores blob data in SQL databases (MySQL or SQLite), and over time, unreferenced blobs may accumulate. + +The tool provides subcommands to: +- Identify unreferenced blobs in SQLblob storage +- Delete garbage blobs to reclaim space +- Process multiple shards of a sharded SQLblob configuration +- Run garbage collection with configurable concurrency + +This tool is specifically for SQLblob storage and is not applicable to other blobstore types like Manifold or S3. + +### Backfill Mapping + +**Location**: `tools/backfill_mapping/` + +**Binary**: `fbcode//eden/mononoke/tools/backfill_mapping:backfill_mapping` + +The backfill_mapping tool populates VCS mapping tables for commits that lack complete mappings. VCS mappings associate Bonsai changesets with their Git or SVN equivalents. + +The tool reads a file containing Mercurial changeset IDs and: +- Resolves them to Bonsai changesets +- Computes the corresponding Git or SVN identifiers +- Inserts the mappings into the appropriate mapping tables + +This tool is used to repair incomplete mapping data or populate mappings after enabling Git or SVN support for an existing repository. + +## Data Operations + +### Executor + +**Location**: `tools/executor/` + +**Binary**: `fbcode//eden/mononoke/tools/executor:executor` + +The executor tool provides a framework for running tasks across multiple repositories or shards. It handles: +- Execution of operations across repository sets +- Sharding and parallel execution +- Task scheduling and coordination + +The executor is used as a framework by other tools that need to operate on multiple repositories concurrently. + +### Import Tools + +**Location**: `tools/import/` + +The import directory contains shared libraries and utilities used by various import tools (blobimport, gitimport, repo_import). These are not standalone tools but provide common functionality for repository imports. + +## When to Use Which Tool + +**For routine operations**: Use the `admin` tool. Most common administrative tasks have corresponding admin subcommands. + +**For importing repositories**: +- Mercurial repositories: Use `blobimport` +- Git repositories: Use `gitimport` +- Complex imports with transformations: Use `repo_import` + +**For data verification**: +- Content-addressed aliases: Use `aliasverify` +- Bonsai ↔ Mercurial consistency: Use `bonsai_verify` +- Git working copy validation: Use `check_git_wc` +- Graph-wide validation: Use `walker` (see [Jobs and Background Workers](3.2-jobs-and-background-workers.md)) + +**For storage maintenance**: +- Packblob operations: Use `packer` +- SQLblob cleanup: Use `sqlblob_gc` +- Storage durability: The `blobstore_healer` job handles this automatically + +**For debugging and testing**: Use `testtool` in non-production environments, or use specific `admin` subcommands in production. + +## Tool Implementation Patterns + +All Mononoke tools follow consistent implementation patterns: + +**Framework**: Tools use `cmdlib/mononoke_app/` as the standard application framework, providing: +- Configuration loading +- Argument parsing +- Repository initialization +- Monitoring and logging setup + +**Repository Access**: Tools use the facet pattern to access repository capabilities. A tool declares which facets it requires, and the framework provides a composed repository object. + +**Execution**: Tools are single-execution programs—they perform their task and exit. Long-running operations should be implemented as jobs rather than tools. + +**Safety**: Tools that modify repository state typically include dry-run modes, validation checks, and logging to prevent accidental data corruption. + +## Integration with Jobs + +Several tools have corresponding background jobs that perform similar operations continuously: + +- `aliasverify` tool verifies specific changesets; continuous verification runs as a scheduled task +- `blobstore_healer` job performs continuous healing; tools can trigger one-off repairs +- `walker` job validates repositories continuously; admin tool provides one-off validation commands + +The distinction is that tools are invoked on-demand for specific operations, while jobs run continuously for ongoing maintenance. + +## Documentation and Help + +All tools provide command-line help via `--help` flags. The admin tool's subcommands each have individual help text describing their options and usage. + +Component-specific documentation: +- Tool README files in component directories (when present) +- Integration tests in `tests/integration/` demonstrate tool usage +- Admin subcommand source in `tools/admin/src/commands/` shows available operations + +## Summary + +Mononoke's tools provide operational capabilities for repository management, data import, verification, and maintenance. The admin tool serves as the primary administrative interface, offering a unified command-line API for most common tasks. Specialized tools handle specific operations like repository import, data verification, and storage maintenance. + +Tools are distinct from servers (which handle client requests) and jobs (which run continuously). Understanding this distinction helps operators choose the appropriate tool for each task and understand how the components fit into Mononoke's overall architecture. + +For background maintenance operations, see [Jobs and Background Workers](3.2-jobs-and-background-workers.md). For server components, see [Servers and Services](3.1-servers-and-services.md). diff --git a/eden/mononoke/docs/3.4-libraries-and-frameworks.md b/eden/mononoke/docs/3.4-libraries-and-frameworks.md new file mode 100644 index 0000000000000..c530f8fac95b4 --- /dev/null +++ b/eden/mononoke/docs/3.4-libraries-and-frameworks.md @@ -0,0 +1,398 @@ +# Libraries and Frameworks + +This document describes the shared libraries and frameworks used across Mononoke applications. These components provide common functionality for building servers, tools, and jobs, and form the foundation layer of Mononoke's architecture as described in the [Architecture Overview](1.3-architecture-overview.md). + +## Introduction + +Mononoke applications are built using a consistent set of libraries that handle common concerns like command-line parsing, configuration loading, monitoring setup, asynchronous operations, SQL database access, and logging. This shared foundation ensures that all Mononoke binaries have consistent behavior and operational characteristics. + +The libraries described in this document fall into several categories: + +- **Application framework** (`cmdlib/mononoke_app/`) - Standard framework for building Mononoke binaries +- **Command-line utilities** (`cmdlib/`) - Argument handling, environment setup, configuration +- **Common utilities** (`common/`) - Async helpers, SQL utilities, logging, and specialized tools +- **Core types** (`mononoke_types/`) - Fundamental data structures +- **Testing utilities** (`tests/utils/`) - Shared testing tools + +## Application Framework: mononoke_app + +The `cmdlib/mononoke_app/` directory contains the standard framework for building Mononoke applications. All Mononoke binaries - servers, tools, and jobs - use this framework to ensure consistent initialization, configuration, and operational behavior. + +### Purpose and Responsibilities + +The framework handles: + +- **Argument parsing** - Defines and parses standard command-line arguments +- **Configuration loading** - Loads repository and system configuration +- **Environment initialization** - Sets up logging, metrics, caching, and database connections +- **Repository access** - Provides initialized repository objects +- **Monitoring setup** - Configures health checks, metrics endpoints, and observability +- **Graceful shutdown** - Handles shutdown signals and cleanup + +### Application Structure + +Applications built with `mononoke_app` follow a standard pattern: + +1. Create a `MononokeAppBuilder` with `FacebookInit` +2. Register any application-specific extensions (optional) +3. Build the application with subcommands (for multi-command tools) or standalone +4. Run the application with monitoring and logging + +The framework provides two main types: + +- **`MononokeAppBuilder`** - Builder for constructing applications +- **`MononokeApp`** - Runtime application object passed to the main function + +### Writing a New Mononoke Binary + +To create a new Mononoke binary: + +**Define the main function:** + +```rust +use anyhow::Result; +use fbinit::FacebookInit; +use mononoke_app::{MononokeApp, MononokeAppBuilder}; + +#[fbinit::main] +fn main(fb: FacebookInit) -> Result<()> { + let app = MononokeAppBuilder::new(fb).build()?; + app.run_with_monitoring_and_logging(async_main, "binary_name", AliveService) +} + +async fn async_main(app: MononokeApp) -> Result<()> { + // Application logic here + Ok(()) +} +``` + +**Access repositories:** + +Applications can open repositories using the repository arguments provided by the framework: + +```rust +use mononoke_app::args::RepoArgs; + +async fn async_main(app: MononokeApp) -> Result<()> { + let repo_args: RepoArgs = app.args()?; + let repo = app.open_repo(&repo_args).await?; + // Use repository facets + Ok(()) +} +``` + +The framework automatically handles argument parsing for repository selection, configuration paths, caching options, logging levels, and other common parameters. + +### Extension System + +Applications can register extensions to add custom argument groups or initialization logic: + +```rust +let app = MononokeAppBuilder::new(fb) + .with_app_extension(CustomExtension::new()) + .build()?; +``` + +Extensions provide additional arguments and can perform initialization when the application starts. Standard extensions include `MonitoringAppExtension` for monitoring setup and `ScrubAppExtension` for blobstore scrubbing capabilities. + +### Subcommands + +For tools with multiple subcommands (like the admin CLI), the framework provides a macro to define subcommands: + +```rust +use mononoke_app::subcommands; + +subcommands! { + mod first_command; + mod second_command; +} +``` + +Each subcommand module defines a `CommandArgs` struct and a `run` function. The framework handles dispatching to the appropriate subcommand based on command-line arguments. + +### Standard Arguments + +The framework automatically provides several categories of command-line arguments: + +**Configuration:** +- `--config-path` - Path to repository configurations +- `--config-tier` - Configuration tier to use + +**Repository Selection:** +- `--repo-name` - Repository name +- `--repo-id` - Repository ID + +**Caching:** +- `--cache-mode` - Caching behavior (local-only, shared, disabled) +- `--cachelib-size` - In-process cache size + +**Storage:** +- `--readonly-storage` - Enable read-only mode +- `--blobstore-options` - Blobstore configuration overrides + +**Logging:** +- `--log-level` - Logging verbosity +- `--scuba-dataset` - Scuba logging dataset + +**Monitoring:** +- `--monitoring-port` - Port for monitoring endpoints + +Applications can add their own arguments alongside these standard options. + +## Command-line Utilities: cmdlib + +The `cmdlib/` directory contains libraries used by `mononoke_app` and available to applications for specific needs. These are organized by functionality: + +### Configuration and Arguments + +**`config_args/`** - Configuration path parsing and loading + +**`commit_id/`** - Commit identifier argument parsing (supports Bonsai IDs, Git hashes, Mercurial hashes, bookmark names) + +**`sharding/`** - Shard selection and filtering arguments + +### Environment and Setup + +**`base_app/`** - Base application framework (used internally by `mononoke_app`) + +**`environment/`** - Mononoke environment setup (database pools, caching, configuration store) + +**`logging/`** - Logging initialization and configuration + +**`log/`** - Structured logging utilities + +### Capabilities + +**`caching/`** - Cachelib initialization and configuration + +**`scrubbing/`** - Blobstore scrubbing support for validation + +**`cross_repo/`** - Cross-repository operation utilities + +**`displaying/`** - Output formatting and display utilities + +### Other Utilities + +**`extensions/`** - Extension framework for adding custom functionality + +**`zk_leader_election/`** - ZooKeeper-based leader election for distributed jobs + +Applications typically do not use these libraries directly - `mononoke_app` provides access to their functionality through the application object and standard arguments. + +## Common Utilities: common/ + +The `common/` directory contains specialized utility libraries used throughout Mononoke. These provide building blocks for asynchronous programming, database access, logging, and other common needs. + +### Asynchronous Programming Utilities + +**`futures_watchdog/`** - Monitors futures and logs warnings when polling takes too long. Helps identify performance problems where futures block for extended periods. + +**`async_limiter/`** - Limits concurrent execution of asynchronous operations. Provides backpressure and resource control. + +**`assembly_line/`** - Sequential processing pipeline for async operations. + +**`running/`** - Utilities for running tasks until termination signals. + +**`yield_stream/`** - Stream utilities that yield control periodically. + +### SQL Database Utilities + +**`sql_construct/`** - Construction utilities for SQL database managers. Provides traits for building database managers from configuration, with support for both MySQL and SQLite backends. + +**`rust/sql_ext/`** (in `common/rust/sql_ext/`) - SQL connection utilities, query builders, and Facebook-specific MySQL features like connection pooling and failover. + +### Logging and Observability + +**`scuba_ext/`** - Extensions for Scuba structured logging. Provides `MononokeScubaSampleBuilder` for consistent logging across Mononoke. + +**`logger_ext/`** - Extensions to the standard logger for Mononoke-specific needs. + +**`ods_counters/`** - ODS metrics counters for operational monitoring. + +**`session_id/`** - Session ID tracking for request correlation. + +**`scribe_ext/`** - Scribe logging integration. + +### Data Structures and Algorithms + +**`dedupmap/`** - Hash map that deduplicates values with the same key. + +**`uniqueheap/`** - Heap data structure that ensures uniqueness of elements. + +**`type_map/`** - Type-indexed map for storing values of different types. + +**`topo_sort/`** - Topological sorting for directed acyclic graphs. + +### Time and Measurement + +**`time_measuring/`** - Utilities for measuring operation duration. + +**`reloader/`** - Periodic reloading of configuration or data. + +### Path and Content Utilities + +**`path_hash/`** - Path hashing utilities. + +**`copy_utils/`** - Utilities for copying data structures. + +### Synchronization and Coordination + +**`rendezvous/`** - Batching of SQL queries with deduplication. + +**`wait_for_replication/`** - Waits for database replication to catch up. + +### Metadata and System Information + +**`metadata/`** - System metadata utilities. + +**`memory/`** - Memory usage tracking and reporting. + +**`connection_security_checker/`** - Connection security validation. + +### Iteration Helpers + +**`iterhelpers/`** - Iterator utility functions and extensions. + +### Thrift Integration + +**`thrift_convert/`** - Conversion utilities between Mononoke types and Thrift types. + +## Shared Utilities: common/rust/shed + +In addition to Mononoke-specific utilities in `common/`, Mononoke uses utilities from `common/rust/shed/`, which provides general-purpose Rust libraries used across multiple Meta projects. + +Key libraries from shed include: + +**`bounded_traversal/`** - Traversal of trees and DAGs with bounded concurrency. Provides functions for processing large graphs in parallel while limiting the number of concurrent operations. Used extensively in Mononoke for walking commit graphs and file trees. + +**`facet/`** - Facet container pattern for trait-based composition. The foundation for Mononoke's repository facet system. + +**`fbinit/`** - Facebook initialization. Required for all Mononoke binaries - provides the `FacebookInit` token that ensures proper initialization of Facebook infrastructure. + +**`cached_config/`** - Configuration system that periodically reloads configuration from external sources. Used for repository configuration and feature flags. + +**`cloned/`** - Macro for cloning variables into closures without verbose syntax. + +**`futures_stats/`** - Statistics collection for futures execution. + +**`stats/`** - Metrics and statistics framework. + +**`scuba_sample/`** - Scuba logging sample builder. + +These libraries are located in `common/rust/shed/` rather than `eden/mononoke/common/` because they are shared across multiple projects at Meta. + +## Core Types: mononoke_types + +The `mononoke_types/` directory defines the fundamental data structures used throughout Mononoke: + +**Bonsai Changesets** - The canonical representation of commits in Mononoke. Includes parent relationships, author information, timestamp, message, and file changes. + +**Content Identifiers** - Content-addressed identifiers using Blake2b hashing. Includes `ContentId` for file contents and `ChangesetId` for commits. + +**File Changes** - Representation of file additions, modifications, and deletions. + +**Paths** - Type-safe path representations with validation (`MPath`, `NonRootMPath`). + +**VCS Types** - Git and Mercurial-specific identifiers and structures for interoperability. + +**Derived Data Types** - Type definitions for various derived data formats. + +These types are used across all layers of Mononoke - from storage through the API layer to protocol servers. They provide a stable interface and ensure type safety throughout the system. + +The types are designed for: + +- **Serialization** - Efficient serialization using `serde` and Thrift +- **Content addressing** - Blake2b hashing for Merkle DAG construction +- **Type safety** - Strong typing to prevent errors (e.g., cannot use a `ContentId` where a `ChangesetId` is expected) + +## Testing Utilities: tests/utils + +The `tests/utils/` directory provides utilities for writing tests: + +**`drawdag`** - A domain-specific language for constructing commit graphs in tests. Allows compact representation of test repositories: + +```rust +// Example usage pattern (simplified) +let commits = drawdag::extend_from_dag( + &repo, + r##" + A-B-C + \ + D-E + "##, +)?; +``` + +This creates a commit graph with branches without verbose setup code. + +**`random`** - Random data generation for testing. Provides functions for creating random file contents, paths, and other test data. + +**Library functions** - Various helper functions for common test setup tasks like creating repositories, initializing storage, and setting up test environments. + +These utilities are used extensively in both unit tests and integration tests to reduce boilerplate and make tests more readable. + +## Usage Patterns + +### Using the Framework + +Most Mononoke development involves either: + +1. **Adding functionality to existing applications** - Modify servers, tools, or jobs that already use `mononoke_app` +2. **Creating new tools** - Build new binaries using the `mononoke_app` framework +3. **Using common utilities** - Import utilities from `common/` for specific needs + +The framework handles initialization and provides access to repositories, so application code can focus on the specific operations needed. + +### Accessing Utilities + +Common utilities are available as dependencies in BUCK files: + +```python +rust_binary( + name = "my_tool", + srcs = glob(["src/**/*.rs"]), + deps = [ + "//eden/mononoke/cmdlib/mononoke_app:mononoke_app", + "//eden/mononoke/common/futures_watchdog:futures_watchdog", + "//eden/mononoke/common/sql_construct:sql_construct", + "//common/rust/shed/bounded_traversal:bounded_traversal", + ], +) +``` + +### Working with Repositories + +Applications access repository facets through the repository object: + +```rust +let blobstore = repo.repo_blobstore(); +let commit_graph = repo.commit_graph(); +let derived_data = repo.repo_derived_data(); +``` + +The `mononoke_app` framework handles repository initialization based on configuration, so applications receive fully initialized repositories. + +## Summary + +Mononoke's libraries and frameworks provide: + +- **Consistent application structure** through `mononoke_app` +- **Standard operational behavior** across all binaries +- **Reusable components** for common tasks +- **Strong typing** through `mononoke_types` +- **Testing support** through shared utilities + +When building new functionality: + +1. Use `mononoke_app` for new binaries +2. Import utilities from `common/` or `shed/` as needed +3. Use `mononoke_types` for data structures +4. Leverage `tests/utils` for testing + +The framework and utilities handle initialization, configuration, monitoring, and other cross-cutting concerns, allowing application code to focus on source control operations. + +For component-specific details: +- See component README files where available +- Review existing applications for usage examples +- Examine BUCK files to understand dependencies +- Refer to rustdoc for API documentation diff --git a/eden/mononoke/docs/4.1-pushrebase.md b/eden/mononoke/docs/4.1-pushrebase.md new file mode 100644 index 0000000000000..0b88ac64067ff --- /dev/null +++ b/eden/mononoke/docs/4.1-pushrebase.md @@ -0,0 +1,265 @@ +# Pushrebase + +This document explains pushrebase, a server-side rebasing mechanism that maintains linear commit history in high-throughput repositories. Pushrebase is a key feature used by both Sapling and Git clients when pushing commits to Mononoke. + +## What is Pushrebase? + +Pushrebase is a server-side operation that automatically rebases commits during a push when the target bookmark has moved forward since the client's base commit. Rather than requiring clients to pull, rebase locally, and retry their push, the server performs the rebase operation automatically for conflict-free changes. + +When a client pushes commits to a bookmark that has moved forward, Mononoke detects whether the changes can be safely rebased. If the pushed commits modify different files than the commits that moved the bookmark forward, the server rebases the pushed commits on top of the current bookmark position and updates the bookmark atomically. + +The result is a linear commit history on the target bookmark without requiring coordination between clients or manual rebase operations. + +## Why Pushrebase Exists + +In repositories with high commit rates and many concurrent developers, bookmark contention becomes a problem. Without pushrebase, when multiple developers attempt to push to the same bookmark: + +1. The first push succeeds +2. Subsequent pushes fail because their base commit is no longer the bookmark's position +3. Each developer must pull the new commits, rebase their changes, and retry +4. With many concurrent pushes, developers may need multiple retry cycles before succeeding + +This retry loop increases push latency and client-side complexity. Pushrebase addresses this by moving the rebase operation to the server, which can perform it atomically during the bookmark update. Clients pushing non-conflicting changes succeed on the first attempt. + +## The Pushrebase Process + +### Client Side + +From the client's perspective, pushrebase is transparent. A client pushes commits to a bookmark using standard Sapling push, or via the Git pushrebase tool provided by Mononoke. The client's base commit may be behind the current bookmark position on the server. + +### Server-Side Operation + +When Mononoke receives a push to a bookmark configured for pushrebase: + +**1. Identify the Rebased Set** + +The server identifies which commits need to be rebased: +- The **pushed set** contains all commits sent by the client +- The **root** is the common ancestor between the pushed commits and the current bookmark position +- The **rebased set** contains the pushed commits that will be rebased on top of the bookmark, which may be smaller than the **pushed set** when merges are involved + +**2. Conflict Detection** + +The server checks whether the rebase can be performed automatically: +- Collect all files modified in the rebased set +- Collect all files modified between the root and the current bookmark position +- Check for path conflicts between these two sets + +A conflict exists if: +- The same file is modified in both sets +- A path in one set is a prefix of a path in the other set (file/directory conflicts) +- Any other manifest-level conflicts exist (delete/modify conflicts) + +**3. Rebase Execution** + +If no conflicts are detected, the server performs the rebase: +- For each commit in the rebased set (starting from the oldest) +- Create a new commit with the same changes but with updated parents +- Update commit metadata (timestamps may be adjusted based on configuration) +- Store the new Bonsai changeset in the blobstore + +**4. Hook Execution** + +Before updating the bookmark, pushrebase runs configured hooks: +- Pre-commit hooks validate the rebased commits +- Hooks can reject the push if validation fails +- Hooks can modify commits (e.g., adding tracking information) +- Transaction hooks can perform additional database updates in the same transaction + +**5. Bookmark Update** + +If hooks pass, the server updates the bookmark: +- Move the bookmark to point to the tip of the rebased commits +- Update VCS mappings (Bonsai ↔ Git/Mercurial hashes) for the rebased commits +- Record the bookmark movement in the update log +- Commit the transaction atomically + +**6. Response to Client** + +The server returns the rebased commits to the client: +- Sapling clients receive obsmarkers mapping original commits to rebased commits +- Git clients receive the new commit hashes +- Clients can update their local state without performing a separate fetch + +### Retry on Concurrent Updates + +If the bookmark moves during the pushrebase operation (due to another concurrent push), the server retries the rebase with the new bookmark position. After a configured number of retries (typically 100), the server fails the push and the client must retry. + +## Conflict-Free Rebases + +Pushrebase only handles conflict-free rebases where no file-level merging is required. This is a conservative approach that ensures correctness - if file contents conflict, human intervention is required. + +The conflict detection is performed at the path level, not the content level. Two commits that modify the same file are considered conflicting even if the changes are to different lines. This conservative approach avoids potential semantic conflicts and unexpected behavior. + +If a push contains conflicts, the server rejects it with a conflict error. The client must: +1. Pull the latest bookmark position +2. Rebase locally (resolving any conflicts manually) +3. Push again (which may succeed via pushrebase if the rebase was clean) + +## Pushrebase Configuration + +Pushrebase behavior is controlled by repository configuration in `metaconfig/`: + +**Per-Repository Settings:** +- `pushrebase.block_merges` - Reject pushes containing merge commits +- `pushrebase.casefolding_check` - Check for case-folding conflicts (important for case-insensitive filesystems) +- `pushrebase.rewritedates` - Update commit timestamps during rebase +- `pushrebase.recursion_limit` - Maximum commit stack depth +- `pushrebase.forbid_p2_root_rebases` - Disallow rebasing certain stacks based on merge commits + +**Per-Bookmark Settings:** +- Bookmarks can be configured to allow or forbid pushrebase +- Some bookmarks may require direct fast-forward pushes instead + +## Pushrebase Hooks + +Pushrebase has a hook mechanism that allows additional optional operations to happen during pushrebase. This is separate from Mononoke's bookmark hook system (documented in `4.3-hooks.md`) although bookmarks hooks are *also* run during pushrebase. Pushrebase hooks run during rebase computation and before the bookmark is updated, allowing: + +- Adding commit tracking information +- Recording the original commit hash (pushrebase mutation mapping) +- Generating sequential commit numbers (GlobalRev) + +Pushrebase hooks are defined in `features/pushrebase/pushrebase_hook/` (the trait) and `features/pushrebase/pushrebase_hooks/` (hook implementations). The `get_pushrebase_hooks()` function in `features/pushrebase/pushrebase_hooks/` constructs the set of hooks to run based on repository configuration. + +## Implementation Architecture + +Pushrebase is implemented across several components: + +**Core Pushrebase (`features/pushrebase/src/`):** +- `do_pushrebase_bonsai()` - Main entry point for pushrebase operation +- Conflict detection and changeset rebasing logic +- Retry loop for concurrent bookmark updates +- Integration with hooks + +**Bookmark Movement (`repo_attributes/bookmarks/bookmarks_movement/`):** +- `pushrebase_onto.rs` - High-level pushrebase operation +- Permission checking and authorization +- Logging and observability +- Integration with bookmark update log + +**Pushrebase Client (`features/pushrebase/client/`):** +- `local.rs` - Direct in-process pushrebase +- `hybrid.rs` - Client that can use local or remote pushrebase +- `facebook/land_service.rs` - Pushrebase via land service microservice + +**Pushrebase Hooks:** +- `features/pushrebase/pushrebase_hook/` - Hook trait definitions +- `features/pushrebase/pushrebase_hooks/` - Hook implementations and registry + +The pushrebase operation is a feature (as described in the [Architecture Overview](1.3-architecture-overview.md)) that composes multiple repository facets: +- `Bookmarks` - Reading and updating bookmarks +- `CommitGraph` - Ancestry queries and graph traversal +- `RepoBlobstore` - Storing rebased changesets +- `HookManager` - Running hooks +- `RepoDerivedData` - Triggering derived data computation + +## Use in Sapling Workflows + +Sapling clients use pushrebase as the default push mode for most bookmarks: + +**Push Operation:** +``` +sl push --to master +``` + +If the local base is behind the server's `master` bookmark, the server automatically rebases the commits. The client receives the rebased commits and updates its local state. + +**Mutation Information:** +Sapling tracks commit rewrites (mutations) caused by pushrebase. When the server rebases commits, it returns obsmarkers that map the original commit hashes to the rebased hashes. This allows Sapling to understand that the local commits and server commits are related. + +**Commit Cloud Integration:** +Pushrebase does not currently integrate with Commit Cloud (Sapling's feature for sharing uncommitted work). When commits are pushrebased, the mutation information is not synchronized across a developer's machines, but rather each seprate machine will discover that the commits have been landed. This is an opportunity for improvement. + +## Use in Git Workflows + +Git clients can also use pushrebase when pushing to Mononoke: + +**Push with Pushrebase:** +Git clients push using the standard Git protocol. If the server is configured for pushrebase on the target branch, the push operation triggers a pushrebase. + +**Mapping to Git:** +When rebased commits are created: +- Mononoke generates corresponding Git commits +- Git commit hashes are computed and stored in `bonsai_git_mapping` +- The server responds to the Git client with the new commit hashes +- The client can update its branch pointers without fetching + +**Differences from Sapling:** +Git does not have a native mutation tracking mechanism like Sapling's obsmarkers. Git clients receive the new commit hashes but do not have a built-in way to track the relationship between original and rebased commits. + +## Pushrebase vs. Fast-Forward Push + +Mononoke supports two primary push modes: + +**Fast-Forward Push:** +- Client's head must be a descendant of the current bookmark position +- No rebasing occurs +- Push fails if the bookmark has moved since the client's pull +- Used for bookmarks requiring strict lineage + +**Pushrebase:** +- Client's head may be based on an outdated bookmark position +- Server rebases conflict-free changes automatically +- Push succeeds as long as there are no file-level conflicts +- Used for high-throughput bookmarks with many concurrent contributors + +The choice between these modes is configured per-bookmark in repository configuration. + +## Pushrebase Mutation Mapping + +The `PushrebaseMutationMapping` facet (in `repo_attributes/pushrebase_mutation_mapping/`) tracks the relationship between original commits and their rebased versions: + +- Original commit hash → Rebased commit hash mapping +- Stored in the metadata database +- Used by Sapling to track commit rewrites +- Queried by SCS and other tools for commit history analysis + +This mapping is populated by a pushrebase transaction hook and is accessible through the SCS API (Source Control Service) for programmatic queries. + +## Observability + +Pushrebase operations are instrumented with metrics and logging: + +**Metrics:** +- `pushrebase.critical_section_success_duration_us` - Time spent in successful critical section +- `pushrebase.critical_section_failure_duration_us` - Time spent in failed critical section +- `pushrebase.critical_section_retries_failed` - Count of retry failures +- `pushrebase.commits_rebased` - Number of commits rebased + +**Logging:** +- Scuba logging for all pushrebase operations +- Bookmark update log records all bookmark movements +- Hook execution is logged for debugging and auditing + +**Debugging:** +The admin tool provides commands for testing and debugging pushrebase: +``` +admin commit pushrebase --help +``` + +## Limitations and Edge Cases + +**Merge Commits:** +Pushrebase can be configured to reject merge commits (`block_merges` configuration). Repositories that maintain strict linear history use this setting. + +**Large Stacks:** +Rebasing very large commit stacks can be slow. The `recursion_limit` configuration prevents unbounded stack depth. + +**Concurrent Updates:** +If a bookmark is updated very frequently, pushrebase may exhaust retries. The client must retry the push operation. + +**Case Conflicts:** +On case-insensitive filesystems, pushrebase can detect potential case conflicts that might cause issues for some clients. + +**Hook Failures:** +If any hook fails during pushrebase, the entire operation is rolled back. The client receives the hook error message. + +## Related Documentation + +- **[Architecture Overview](1.3-architecture-overview.md)** - How pushrebase fits into the feature layer +- **[Repository Facets](2.2-repository-facets.md)** - Facets used by pushrebase (bookmarks, commit graph, etc.) +- **[Bookmarks](TBD)** - Bookmark management and update semantics +- **[Hooks](4.3-hooks.md)** - Hook system integrated with pushrebase +- **[Land Service](3.1-servers-and-services.md)** - Microservice for coordinating pushrebase operations + +The pushrebase implementation is in `features/pushrebase/` and its integration with bookmark movement is in `repo_attributes/bookmarks/bookmarks_movement/src/pushrebase_onto.rs`. diff --git a/eden/mononoke/docs/4.2-cross-repo-sync.md b/eden/mononoke/docs/4.2-cross-repo-sync.md new file mode 100644 index 0000000000000..6b17e68b5eaa5 --- /dev/null +++ b/eden/mononoke/docs/4.2-cross-repo-sync.md @@ -0,0 +1,277 @@ +# Cross-Repository Sync + +This document explains Mononoke's cross-repository synchronization system—the framework for automatically synchronizing commits between different repositories. + +## What is Cross-Repo Sync? + +Cross-repo sync is a system that maintains bidirectional synchronization between repositories. It automatically replicates commits from one repository to another, transforming file paths and commit metadata as needed to match each repository's structure. + +The most common use case is synchronizing between a large repository (monorepo) and smaller project-specific repositories. When a developer pushes to a small repository, those changes are automatically synced to the corresponding location in the large repository. Conversely, changes in the large repository can be synced back to the small repositories. + +Cross-repo sync handles: +- Path transformation (files in different locations between repos) +- Bookmark mapping (branch names may differ) +- Commit metadata preservation +- History linearization through pushrebase +- Merge commit handling (with limitations) + +## Sync Directions + +Cross-repo sync operates in two directions: + +**Forward Sync (Small → Large)** +- Commits from small repositories are synced into the large repository +- Each small repo's files are placed in a designated prefix within the large repo +- This is the primary direction when small repos are the source of truth +- Implemented by the x-repo sync job + +**Backward Sync (Large → Small)** +- Commits from the large repository are synced to small repositories +- Only changes affecting the small repo's prefix are synced +- Maintains the ability to develop in either repository +- Implemented by the backsyncer + +The relationship between repositories is configured rather than inferred from structure. A repository can participate in multiple sync relationships. + +## Commit Transformation + +When syncing commits between repositories, the sync system transforms them to match the target repository's structure. + +### Path Transformation + +Files are moved according to configured path mappings. Each small repository has a map defining how paths are transformed: + +**Example transformation:** +- Small repo path: `src/foo.rs` +- Large repo path: `projects/myproject/src/foo.rs` + +The mapping is defined in the commit sync configuration using prefix replacements. More complex transformations can be expressed through multiple mapping entries or by using the default action configuration. + +### Commit Rewriting + +Commits are rewritten during sync using the commit transformation library (`features/commit_transformation/`). The rewriting process: + +1. Loads the source commit +2. Applies path transformations to file changes +3. Remaps parent commits (already synced) +4. Preserves author, timestamp, and commit message +5. Stores the rewritten commit in the target repository + +Commits that would be empty after path transformation (no files in the mapped paths) are handled specially. These may be recorded as "not a sync candidate" or mapped to an equivalent working copy ancestor. + +### Special Cases + +**Merge Commits** - Supported with limitations. Both parents must already be synced. Some complex merge scenarios are not supported and will cause sync to fail. + +**Copy/Move Information** - File copy and move metadata is preserved when both source and destination paths map to the target repository. + +**Git Submodules** - When configured, submodule expansion can be performed during sync, converting submodule pointers into expanded directory contents. + +## Sync Process + +Cross-repo sync operates by tailing the bookmark update log of the source repository. + +### Forward Sync Process + +The x-repo sync job (`features/commit_rewriting/mononoke_x_repo_sync_job/`) performs forward sync: + +1. **Tail Bookmark Updates** - Monitor bookmark update log for changes +2. **Find Unsynced Ancestors** - Identify commits not yet synced to target repo +3. **Sync in Order** - Process commits topologically (parents before children) +4. **Transform Commits** - Rewrite each commit for the target repository +5. **Pushrebase** - Land synced commits onto target bookmarks via pushrebase +6. **Record Mapping** - Store source-to-target commit mapping +7. **Update Counter** - Record progress for resumption + +Common pushrebase bookmarks (configured as `common_pushrebase_bookmarks`) receive special handling. These are bookmarks where pushrebase is used to maintain linear history in both repositories. + +### Backward Sync Process + +The backsyncer (`features/commit_rewriting/backsyncer/`) performs backward sync: + +1. **Tail Large Repo** - Monitor bookmark updates in the large repository +2. **Filter Changes** - Select commits affecting the small repo's mapped paths +3. **Transform Commits** - Rewrite commits removing unrelated paths +4. **Sync to Small Repo** - Push transformed commits to small repository +5. **Record Mapping** - Store bidirectional mapping + +Backsyncer can run continuously or in catch-up mode to process historical commits. + +### Synced Commit Mapping + +All synced commits are recorded in the synced commit mapping database (`features/commit_rewriting/synced_commit_mapping/`). This mapping stores: + +- Source repository and changeset ID +- Target repository and changeset ID +- Sync config version used +- Sync outcome (rewritten, equivalent working copy, not a sync candidate) + +The mapping is bidirectional, allowing queries in either direction. It is consulted before syncing to avoid duplicate work and to remap parent commits correctly. + +## Commit Sync Outcomes + +When a commit is synced, the result is recorded as a commit sync outcome: + +**RewrittenAs** - The commit was transformed and created as a new commit in the target repository. This is the most common outcome. + +**EquivalentWorkingCopyAncestor** - The commit would be empty after transformation (no files in mapped paths), so it maps to an ancestor commit with the same working copy state. + +**NotSyncCandidate** - The commit should not be synced to the target repository. This is recorded when all file changes are outside mapped paths. + +Multiple source commits may map to the same target commit when they only affect unmapped paths. The plural commit sync outcome type represents this many-to-one relationship. + +## Configuration + +Cross-repo sync is configured through the `CommitSyncConfig` structure in repository metadata. + +### Commit Sync Config Structure + +Each sync relationship defines: + +**Large Repository** - The repository ID of the large (monorepo) repository + +**Small Repositories** - Map of small repository IDs to their configurations, each containing: +- **Default Action** - What to do with paths not explicitly mapped +- **Path Map** - Prefix replacements (small repo path → large repo path) +- **Submodule Config** - Git submodule handling if applicable + +**Common Pushrebase Bookmarks** - Bookmarks where changes from small repos are pushed via pushrebase + +**Version Name** - Identifies this configuration version + +### Configuration Versioning + +Sync configurations are versioned using `CommitSyncConfigVersion`. This allows the sync rules to evolve over time: + +- New path mappings can be added in a new version +- Different commits may be synced using different config versions +- The version is recorded in the synced commit mapping +- Live config can change while preserving historical mappings + +Configuration is managed through `live_commit_sync_config` (`features/commit_rewriting/live_commit_sync_config/`), which provides access to both current and historical configuration versions. + +### Bookmark Mapping + +Small repository bookmarks can be mapped to large repository bookmarks with a configured prefix. For example: +- Small repo bookmark: `main` +- Large repo bookmark: `projectname/main` + +The bookmark prefix is configured per small repository in the permanent configuration. + +## Repository Integration + +Cross-repo sync is integrated into repositories through the `repo_cross_repo` facet (`repo_attributes/repo_cross_repo/`). This facet provides: + +- **Synced Commit Mapping** - Database of synced commits +- **Live Commit Sync Config** - Current and historical sync configurations +- **Sync Lease** - Coordination to prevent concurrent sync attempts +- **Submodule Dependencies** - Repository IDs of submodule dependencies + +The facet is used by both sync jobs and by other repository operations that need to query or update sync state. + +## Use Cases + +Cross-repo sync serves several use cases: + +**Monorepo and Project Repos** - A large monorepo contains many projects. Each project can have a dedicated small repository. Developers can commit to either repository, and changes are automatically synchronized. + +**Code Sharing** - Common code can be shared between repositories. Changes in the shared code are propagated to all repositories that include it. + +**Migration** - Repositories can be gradually migrated into or out of a monorepo while maintaining dual operation during the transition period. + +**Megarepo Operations** - Initial import of repositories into a megarepo uses cross-repo sync mechanisms to establish the initial mappings and history. See `megarepo_api/` for megarepo-specific operations. + +**Directory Isolation** - A large repository can be split into smaller repositories by directory, allowing teams to work in isolated repositories while maintaining the option to work in the full repository. + +## Jobs and Tools + +### X-Repo Sync Job + +The primary sync job (`features/commit_rewriting/mononoke_x_repo_sync_job/`) runs continuously to sync commits from small repositories to the large repository. It can be run in sharded mode across multiple processes for scalability. + +**Operation:** +- Tails bookmark update log +- Processes commits in topological order +- Uses pushrebase for common bookmarks +- Records progress in mutable counters +- Resumes from last processed position on restart + +### Backsyncer + +The backsyncer (`features/commit_rewriting/backsyncer/`) syncs commits from large repository to small repositories. It can run continuously or in catch-up mode. + +**Modes:** +- Continuous tailing of large repo bookmarks +- Catch-up mode to process historical commits +- Validation mode to verify sync correctness + +### Admin Tools + +The admin CLI (`tools/admin/`) provides commands for cross-repo sync operations: +- Manually triggering sync of specific commits +- Querying synced commit mappings +- Validating working copy equivalence +- Inspecting sync configuration + +## Limitations and Constraints + +The cross-repo sync system has several limitations: + +**Merge Commits** - Some merge commit scenarios are not supported. Merges where both parents have not been synced will fail. + +**Root Commits** - Root commits (commits with no parents) and their descendants may not sync unless merged into a main line of development. + +**Path Conflicts** - If path transformations would create conflicts (multiple source paths mapping to the same target path), sync will fail. + +**Bookmark Filters** - Not all bookmarks are synced. Only explicitly configured bookmarks participate in sync. + +**Sequential Processing** - Commits must be synced in topological order, which can limit parallelism. + +These limitations are documented in the sync job source code and are checked during sync operations. + +## Performance Considerations + +Cross-repo sync is designed for continuous operation with commits arriving at high rates: + +**Incremental Sync** - Only new commits since the last sync are processed, not the entire repository history. + +**Batching** - Multiple commits can be processed in a single sync iteration when catching up. + +**Leasing** - Sync lease prevents multiple servers from syncing the same commits concurrently, avoiding wasted work. + +**Derived Data** - Derived data can be computed asynchronously after sync completes, not blocking the sync operation. + +**Caching** - Live commit sync config is cached to avoid repeated configuration lookups. + +**Sharding** - The sync job can be sharded across multiple processes, each handling a subset of repositories. + +## Monitoring and Validation + +Cross-repo sync includes validation mechanisms: + +**Working Copy Verification** - The `verify_working_copy` function confirms that synced commits have identical working copies (after path transformation). + +**Bookmark Diff** - The `find_bookmark_diff` function identifies discrepancies between bookmarks in source and target repositories. + +**Commit Validator** - The commit validator (`features/commit_rewriting/commit_validator/`) checks that forward and backward sync produce consistent results. + +**Scuba Logging** - All sync operations are logged to Scuba for monitoring and debugging. + +Validation can detect sync configuration errors, sync job failures, or unexpected commit transformations. + +## Related Documentation + +* [Architecture Overview](1.3-architecture-overview.md) - Cross-repo sync in system context +* [Repository Facets](2.2-repository-facets.md) - RepoCrossRepo facet +* [Pushrebase](4.1-pushrebase.md) - How commits are landed during sync +* [Jobs and Background Workers](3.2-jobs-and-background-workers.md) - Sync job operational context + +Component-specific documentation: +- `features/cross_repo_sync/` - Core sync library implementation +- `features/commit_transformation/` - Commit rewriting logic +- `features/commit_rewriting/mononoke_x_repo_sync_job/` - Forward sync job +- `features/commit_rewriting/backsyncer/` - Backward sync job +- `features/commit_rewriting/synced_commit_mapping/` - Mapping database +- `features/commit_rewriting/live_commit_sync_config/` - Configuration management +- `repo_attributes/repo_cross_repo/` - Repository facet +- `megarepo_api/` - Megarepo operations diff --git a/eden/mononoke/docs/4.3-hooks.md b/eden/mononoke/docs/4.3-hooks.md new file mode 100644 index 0000000000000..fa49eeaaa8c39 --- /dev/null +++ b/eden/mononoke/docs/4.3-hooks.md @@ -0,0 +1,258 @@ +# Hooks + +This document explains Mononoke's repository hooks system. Hooks are configurable checks that execute during bookmark updates to enforce repository policies, validate commits, and prevent problematic changes from being accepted. + +## What Are Hooks? + +Hooks are validation mechanisms that run when commits become ancestors of public bookmarks (branches). When a developer pushes commits or a bookmark is moved, Mononoke executes the configured hooks for that bookmark. If any hook rejects a commit, the entire push operation fails. + +Hooks serve as the enforcement point for repository policies. They can validate commit messages, check file contents, enforce naming conventions, limit file sizes, verify permissions, and implement custom business logic. Unlike client-side hooks that can be bypassed, Mononoke's server-side hooks execute on every push. + +### Hooks vs. Pushrebase Hooks + +**Important:** This document describes Mononoke's repository **Hooks** system (bookmark hooks, changeset hooks, and file hooks). There is a separate system called **Pushrebase hooks** that serves a different purpose. While the naming is similar, these are distinct systems: + +- **Hooks** (this document) - Policy enforcement hooks that validate commits during bookmark movements. They run before the bookmark is updated and can reject pushes that violate repository policies. +- **Pushrebase hooks** (documented in [Pushrebase](4.1-pushrebase.md#pushrebase-hooks)) - Specialized hooks that run during the server-side rebase computation to add metadata like commit tracking information or GlobalRev numbers. + +Both systems run during pushrebase operations, but they serve different purposes and execute at different points in the process. + +## Hook Types + +Mononoke supports three types of hooks, each operating at a different granularity: + +### Bookmark Hooks + +Bookmark hooks execute once per bookmark movement. They have access to the bookmark being updated and the target changeset. Bookmark hooks typically validate bookmark-specific policies such as ensuring certain bookmarks are only moved by authorized services or verifying that tags follow specific naming conventions. + +The `BookmarkHook` trait is defined in `repo_attributes/hook_manager/hook_manager/src/lib.rs`. + +### Changeset Hooks + +Changeset hooks execute once per changeset being pushed. They have access to the full changeset metadata, including author information, commit message, and the set of file changes. Changeset hooks typically validate commit message format, check for merge commits when linear history is required, verify author email domains, or enforce commit size limits. + +The `ChangesetHook` trait is defined in `repo_attributes/hook_manager/hook_manager/src/lib.rs`. + +### File Hooks + +File hooks execute once per file change (addition, modification, or deletion). They receive the file path and optionally the file content. File hooks typically enforce filename restrictions, check file content for prohibited patterns, limit individual file sizes, validate file permissions, or check for specific file types. + +The `FileHook` trait is defined in `repo_attributes/hook_manager/hook_manager/src/lib.rs`. + +## Hook Manager Facet + +The `HookManager` facet (`repo_attributes/hook_manager/`) manages hook registration, configuration, and execution. Like other repository facets, functions that need to run hooks declare their dependency on the hook manager through trait bounds. + +The hook manager maintains: +- Registered hooks and their configurations +- Bookmark-to-hook mappings (which hooks run for which bookmarks) +- Permission checkers for hook bypass authorization +- Scuba logging for hook execution metrics + +Hook execution is integrated into bookmark movement operations in `repo_attributes/bookmarks/bookmarks_movement/`. When a bookmark is moved (during a push, pushrebase, or other operation), the bookmark movement code invokes the hook manager to run all applicable hooks. + +## Hook Configuration + +Hooks are configured per-repository in the repository's metadata configuration. Configuration includes: + +**Hook Registration** +Each hook has a name and a type (bookmark, changeset, or file). The hook implementation is specified, along with any hook-specific configuration options. + +**Bookmark Assignment** +Hooks are associated with bookmarks or bookmark patterns. A hook can be assigned to: +- Specific bookmarks by name +- Bookmarks matching a regular expression +- Bookmarks not matching a regular expression (inverse regex) + +This allows different branches to have different validation requirements. Production branches might have stricter hooks than development branches. + +**Hook Options** +Hooks can accept configuration parameters in JSON format or through typed configuration maps (strings, integers, lists). For example, a file size limit hook would be configured with the maximum allowed size, while a filename pattern hook would be configured with the patterns to allow or deny. + +**Log-Only Mode** +Hooks can be configured in log-only mode. In this mode, hook rejections are logged but do not prevent the push from succeeding. This is useful for testing new hooks before enforcing them or for gathering metrics on policy violations without disrupting developer workflows. + +**Bypass Settings** +Hooks can be configured with bypass conditions. For example, hooks might be bypassed for pushes authored by specific services rather than users, or for cross-repository sync operations. + +The `HookConfig` struct in `metaconfig/types/src/lib.rs` defines the configuration structure. + +## Hook Execution + +When a bookmark is updated, the hook execution process follows these steps: + +**Hook Selection** +The hook manager determines which hooks apply to the bookmark being moved. This includes hooks explicitly assigned to the bookmark and hooks matching the bookmark through regular expressions. + +**Changeset and File Enumeration** +For changeset and file hooks, the system identifies which changesets are new (not already ancestors of other public bookmarks). For each changeset, file changes are enumerated. + +**Hook Invocation** +Hooks are invoked asynchronously. Multiple hooks can run concurrently. Each hook receives: +- Context (containing permissions, logging, tracing) +- Repository access (through the `HookRepo` interface) +- Changeset or file data +- Metadata about the push source and author + +**Result Collection** +Each hook returns either acceptance or rejection. Rejections include a short description (for grouping similar failures) and a long description (for presenting to the user with guidance on fixing the issue). + +**Decision** +If any hook rejects, the entire bookmark movement fails. All rejections are collected and returned to the client. If all hooks accept (or are in log-only mode), the bookmark movement proceeds. + +Hook execution is logged to Scuba for monitoring and analysis. Metrics include execution time, success/failure status, and any rejection messages. + +## Built-in Hook Implementations + +Mononoke includes numerous hook implementations in `features/hooks/src/implementations/`. These provide common validation patterns: + +**File Validation** +- `deny_files` - Blocks files matching specific path patterns +- `block_files` - Prevents specific files from being modified +- `block_content_pattern` - Blocks file content matching specific patterns +- `no_bad_filenames` - Rejects filenames with problematic characters +- `no_questionable_filenames` - Validates filenames to prevent questionable characters +- `no_windows_filenames` - Prevents filenames invalid on Windows +- `no_insecure_filenames` - Blocks potentially dangerous filenames +- `block_invalid_symlinks` - Validates symbolic link targets + +**Size Limits** +- `limit_filesize` - Enforces maximum individual file size +- `limit_commit_size` - Limits total size of changes in a commit +- `limit_directory_size` - Limits total size of a directory +- `limit_path_length` - Enforces maximum path length +- `limit_subtree_op_size` - Limits the size of subtree operations based on file count and path depth + +**Commit Message Validation** +- `require_commit_message_pattern` - Requires commit messages to match a pattern +- `block_commit_message_pattern` - Rejects commit messages matching a pattern +- `limit_commit_message_length` - Enforces commit message length limits + +**Commit Structure** +- `block_merge_commits` - Prevents merge commits (enforces linear history) +- `block_unclean_merge_commits` - Allows only clean merges +- `block_empty_commit` - Rejects empty commits + +**Repository Structure** +- `block_accidental_new_bookmark_creation` - Prevents unintended bookmark creation +- `block_new_bookmark_creations_by_name` - Restricts bookmark creation by name pattern +- `limit_submodule_edits` - Restricts changes to Git submodules + +**Binary and Executable Files** +- `no_executable_binaries` - Prevents executable binary files + +**Git-Specific** +- `block_unannotated_tags` - Requires annotated tags +- `limit_tag_updates` - Restricts tag modifications + +**LFS** +- `missing_lfsconfig` - Validates Git LFS configuration + +**Testing and Debugging** +- `always_fail_changeset` - Always rejects commits (used for testing hook execution) + +Additional implementations exist in `features/hooks/src/facebook/implementations/` for Facebook-internal policies. + +## Hook Bypass + +Under certain conditions, hooks can be bypassed: + +**Administrative Bypass** +Users who are members of the admin group or have the `write_no_hooks` permission can set the `BYPASS_ALL_HOOKS` pushvar to skip hook execution. This is used for emergency fixes or automated processes that need to bypass validation. Bypass usage is logged to Scuba for auditing. + +**Service Pushes** +Hooks can distinguish between pushes authored by users and pushes authored by services. Many hooks skip execution for service pushes, as service-originated commits are considered trusted. Hooks that still need to run for services (such as data integrity validators) can check the `PushAuthoredBy` parameter. + +**Cross-Repository Sync** +Hooks can distinguish between commits pushed directly to a repository and commits synchronized from another repository (push-redirected commits). Hooks may skip execution for synchronized commits, as they were already validated in the source repository. + +The bypass logic is implemented in `repo_attributes/bookmarks/bookmarks_movement/src/hook_running.rs`. + +## Hook Development + +New hooks are implemented by: + +1. Creating a struct that implements one of the hook traits (`BookmarkHook`, `ChangesetHook`, or `FileHook`) +2. Implementing the `run` method to perform validation and return `HookExecution::Accepted` or `HookExecution::Rejected` +3. Registering the hook with the hook manager during repository initialization +4. Configuring the hook in repository metadata + +Hook implementations receive a `HookRepo` interface providing access to repository data without requiring all repository facets. This interface is defined in `repo_attributes/hook_manager/hook_manager/src/repo.rs` and provides methods for fetching file contents, listing changesets, and accessing other data needed for validation. + +Hooks should be stateless—all necessary state should be passed through configuration. This allows hook configuration to be changed without code deployment. + +## Integration with Repository Operations + +Hooks are integrated into several repository operations: + +**Pushrebase** +When commits are landed via pushrebase (`features/pushrebase/`), hooks run before the server-side rebase is computed. Hooks validate the original pushed commits, and if they reject, the pushrebase is aborted before any rebasing occurs. If hooks pass, the server-side rebase proceeds, and then pushrebase hooks (a separate system) add metadata to the rebased commits before the bookmark is moved. + +**Bookmark Movement** +Direct bookmark updates (in `repo_attributes/bookmarks/bookmarks_movement/`) run hooks after validating other constraints (permissions, restrictions) but before updating the bookmark. This ensures all validations occur before the repository state changes. + +**Landing Service** +The land service (`servers/land_service/`) coordinates hook execution for landing operations, serializing execution when necessary to prevent conflicts. + +Hooks execute within a transaction boundary—if they reject, no repository state is modified. + +## Observability + +Hook execution is instrumented for monitoring and debugging: + +**Scuba Logging** +Each hook execution is logged with: +- Hook name and type +- Repository and bookmark +- Changeset hash +- Execution time +- Success or rejection status +- Rejection details + +**Bypassed Commits** +When hooks are bypassed (via admin override or configuration), the bypassed commits are logged separately to a Scuba table for security auditing. + +**Performance Metrics** +Hook execution time is tracked to identify slow hooks. Hooks that significantly impact push latency can be optimized or moved to asynchronous validation. + +## Relationship to Other Features + +Hooks interact with several Mononoke features: + +**Pushrebase** +Hooks validate commits during pushrebase operations, but they run **before** the server-side rebase occurs. When a push triggers pushrebase, hooks execute on the original pushed commits before rebasing happens. If any hook rejects a commit, the pushrebase is aborted and the bookmark remains unchanged. This validation-before-rebase approach ensures that policy violations are caught early in the process. + +After the regular hooks pass and the rebase computation completes, the separate pushrebase hooks system (documented in [Pushrebase](4.1-pushrebase.md#pushrebase-hooks)) runs to add metadata to the rebased commits. + +**Cross-Repository Sync** +Hooks can be configured to skip execution for commits synchronized from other repositories, avoiding duplicate validation when commits flow through multiple repositories. + +**Derived Data** +Hooks execute on the write path and do not wait for derived data. They operate on Bonsai changesets and file content, which are available immediately. If a hook needs indexed data, it must either compute it inline or rely on pre-existing derived data. + +**Permissions** +Hook bypass requires specific permissions (admin group membership or `write_no_hooks` action). The hook manager integrates with the permission system to verify bypass authorization. + +## Summary + +Hooks provide server-side policy enforcement in Mononoke. The hook system: + +**Three Hook Types** - Bookmark, changeset, and file hooks operate at different granularities. + +**Configurable Execution** - Hooks are assigned to bookmarks or bookmark patterns, allowing different validation for different branches. + +**Rejection with Feedback** - Hooks that reject provide descriptions to help developers fix issues. + +**Bypass Mechanisms** - Administrative bypass and service push detection allow flexibility when needed. + +**Integrated Execution** - Hooks execute during bookmark movement operations, running after other validations but before repository state changes. + +**Built-in Implementations** - Mononoke includes many standard hooks for common validation patterns. + +Hooks are implemented in `features/hooks/` and managed by the `HookManager` facet in `repo_attributes/hook_manager/`. Integration with bookmark movement is in `repo_attributes/bookmarks/bookmarks_movement/`. + +## Related Documentation + +- [Repository Facets](2.2-repository-facets.md) - How the HookManager facet fits into the repository composition +- [Pushrebase](4.1-pushrebase.md) - How hooks integrate with server-side rebasing +- [Architecture Overview](1.3-architecture-overview.md) - How hooks fit into the overall system diff --git a/eden/mononoke/docs/4.4-redaction.md b/eden/mononoke/docs/4.4-redaction.md new file mode 100644 index 0000000000000..03b53b65232ed --- /dev/null +++ b/eden/mononoke/docs/4.4-redaction.md @@ -0,0 +1,155 @@ +# Content Redaction + +This document explains Mononoke's content redaction system, which allows removal of sensitive content from repositories while maintaining history integrity. Redaction is implemented at the blobstore layer and operates independently of the version control data model. + +## What is Redaction? + +Content redaction is the process of preventing access to specific file contents in a repository. When content is redacted, the repository history remains intact—commits, file paths, and metadata are preserved—but attempts to access the actual file contents are blocked. + +Redaction addresses several scenarios: +- Accidentally committed credentials or secrets +- Sensitive data that must be removed for compliance or security reasons +- Content that violates policies or legal requirements +- Data subject to removal requests + +Unlike commit rewriting or history modification, redaction does not alter the repository's commit graph. The redacted content remains stored in the blobstore but is marked as inaccessible. + +## How Redaction Works + +Redaction is implemented through the `RedactedBlobstore` decorator in `blobstore/redactedblobstore/`. This decorator wraps the repository's blobstore and intercepts all access attempts. + +### Blobstore-Level Enforcement + +The `RedactedBlobstore` maintains a mapping of redacted blobstore keys (content IDs) to metadata. When a blob is requested: + +1. **Access Check** - The blobstore key is checked against the redacted content list +2. **Decision** - If the key is redacted: + - **Enforce mode** - Returns a `Censored` error containing the reason for redaction + - **Log-only mode** - Logs the access attempt but allows the operation to proceed +3. **Audit** - All access attempts to redacted content are logged to Scuba with session information, username, and enforcement status +4. **Passthrough** - If not redacted, the request proceeds to the underlying blobstore + +This approach intercepts access at the storage layer regardless of which protocol (Sapling, Git, SCS) or operation (clone, fetch, blame) is being used. + +### Redaction Configuration + +Redaction configuration consists of two components stored separately: + +**Redaction Sets** - Configuration stored in Configerator (Meta's configuration management system) at a path specified in repository configuration. Each redaction set contains: +- `reason` - Reference to a task or SEV explaining why content was redacted +- `id` - RedactionKeyListId pointing to the list of redacted keys in the blobstore +- `enforce` - Boolean controlling whether access is denied (true) or only logged (false) + +**Redaction Key Lists** - Lists of blobstore keys stored in a special `RedactionConfigBlobstore`. Each list is a serialized `RedactionKeyList` containing the actual content IDs to redact. + +The configuration is reloaded periodically (every 60 seconds) from Configerator, allowing redaction policy updates without server restarts. + +### Content Addressing + +Redaction operates on content IDs (Blake2b hashes of file contents), not file paths or commits. This means: +- The same content appearing in multiple files or commits is redacted everywhere +- Renaming or moving a file does not bypass redaction +- Different file contents at the same path are treated independently + +## The Redaction Process + +Redacting content involves multiple steps performed using the admin CLI tool in `tools/admin/`. + +### Identifying Content to Redact + +The first step is determining which content needs redaction. The admin tool provides commands for this: + +**From file paths** - Given a commit and file paths, the tool: +1. Derives the fsnode manifest for the specified commit +2. Looks up content IDs for the specified paths +3. Checks whether any content exists in the main bookmark +4. Creates a redaction key list if checks pass + +**From blobstore keys** - For cases where content IDs are already known, a key list can be created directly from blobstore keys. + +**Listing redacted content** - Given a commit, the tool can search for all redacted file paths by comparing content IDs against the current redaction configuration. + +### Creating a Redaction Key List + +The `create-key-list` command creates a `RedactionKeyList` and stores it in the redaction config blobstore: + +``` +admin redaction create-key-list -i file1.txt file2.txt +``` + +This command: +1. Resolves file paths to content IDs in the specified commit +2. Validates that content is not reachable from the main bookmark (unless `--force` is used) +3. Creates a `RedactionKeyList` containing the content IDs +4. Stores the list in the redaction config blobstore +5. Returns a `RedactionKeyListId` (a Blake2b hash of the serialized list) + +The key list can optionally be written to a file using `--output-file`. + +### Activating Redaction + +Creating a key list does not activate redaction. The list ID must be added to the Configerator redaction configuration: + +1. Add a new `RedactionSet` entry to the repository's redaction configuration in Configerator +2. Specify the `RedactionKeyListId`, a `reason` (task or SEV), and `enforce` status +3. Submit and deploy the configuration change +4. Mononoke servers reload the configuration within 60 seconds and begin enforcing the redaction + +### Log-Only Mode + +Setting `enforce: false` in a redaction set enables log-only mode. Access to redacted content is permitted but logged. This allows: +- Testing redaction before enforcement +- Monitoring access to sensitive content +- Gradual rollout of redaction policies + +Access logs include the session ID, username (if available), operation type, and enforcement status. + +## Access Control and Permissions + +Redaction enforcement operates independently of repository permissions. Even users with read access to a repository cannot access redacted content. + +The `CoreContext` carries session information used for audit logging but does not grant exceptions to redaction. There is no mechanism to selectively grant access to redacted content based on user identity. + +For operational purposes, the `RedactedBlobstore` provides an `as_inner_unredacted()` method that returns the underlying blobstore, but this is used only by internal tools that explicitly need to bypass redaction for maintenance operations. + +## Limitations and Considerations + +**Metadata Preservation** - Redaction affects only file contents. Commit messages, file paths, author information, and other metadata remain visible. If metadata itself contains sensitive information, redaction alone is insufficient. + +**Historical Commits** - Commits that include redacted content remain in the repository. The commit graph is not modified. Only access to the file contents is prevented. + +**Content-Addressed Nature** - Redaction applies to content IDs. If the same content was previously accessible, copies may exist in caches (memcache, cachelib, local filesystems). Cache expiration eventually removes these copies, but immediate removal is not guaranteed. + +**Derived Data** - Some derived data types (like Git trees or Mercurial manifests) may reference redacted content IDs. Accessing these derived data structures succeeds, but attempts to fetch the actual file contents fail. + +**No Content Deletion** - Redaction prevents access but does not delete content from the blobstore. The blobs remain stored and continue consuming storage space. Physical deletion would require separate blobstore cleanup operations. + +**Configuration Propagation** - Changes to redaction configuration take up to 60 seconds to propagate to all servers. During this window, some servers may enforce redaction while others do not. + +## Implementation Details + +The redaction system is implemented across several components: + +**Blobstore Decorator** (`blobstore/redactedblobstore/`) - The `RedactedBlobstore` implements the `Blobstore` trait and wraps another blobstore implementation. It maintains the redacted content map and performs access checks. + +**Feature Library** (`features/redaction/`) - Provides high-level functions for creating and managing redaction key lists. Used by the admin tool. + +**Admin Commands** (`tools/admin/src/commands/redaction/`) - Command-line interface for redaction operations: +- `create-key-list` - Create key list from file paths in a commit +- `create-key-list-from-ids` - Create key list from blobstore keys directly +- `fetch-key-list` - Retrieve a key list by ID +- `list` - List redacted paths in a commit + +**Type Definitions** (`mononoke_types/`) - Defines `RedactionKeyList` and `RedactionKeyListId` types for serialization and content addressing. + +**Configuration** (`metaconfig/types/`) - Defines `RedactionConfig` structure specifying the blobstore for key lists and the Configerator location for redaction sets. + +The `RedactedBlobstore` is inserted into the blobstore decorator stack by the repository factory, positioned to intercept all content access regardless of caching or other decorators. + +## Related Documentation + +* [Storage Architecture](2.4-storage-architecture.md) - Blobstore decorator pattern and storage stack +* [Architecture Overview](1.3-architecture-overview.md) - Data flow and request handling +* [Navigating the Codebase](1.4-navigating-the-codebase.md) - Finding redaction-related code + +Component-specific implementation details are located in `blobstore/redactedblobstore/` and `features/redaction/`. diff --git a/eden/mononoke/docs/4.5-microwave.md b/eden/mononoke/docs/4.5-microwave.md new file mode 100644 index 0000000000000..7e01360e64794 --- /dev/null +++ b/eden/mononoke/docs/4.5-microwave.md @@ -0,0 +1,209 @@ +# Microwave + +This document explains Mononoke's microwave cache warming system—a mechanism for preloading caches with repository data before they are needed. + +## What is Microwave? + +Microwave (`features/microwave/`) is a cache warming system that addresses the cold start problem in Mononoke deployments. When a Mononoke server starts with empty caches, initial requests experience higher latency as data is fetched from backend storage and loaded into cache. Microwave preloads caches with repository state, reducing this initial latency. + +The system consists of two components: + +**Builder** (`features/microwave/builder/`) - Creates snapshots of repository state that can be stored and later used for cache warming. The builder binary runs as an offline job. + +**Preloader** (`features/microwave/src/lib.rs`) - Loads stored snapshots and populates server caches during startup or on-demand. + +Microwave integrates with the broader cache warmup system (`features/cache_warmup/`) which coordinates various cache preloading activities. + +## How Cache Warming Works + +Microwave operates in two phases: snapshot creation and snapshot loading. + +### Snapshot Creation + +The microwave builder creates a snapshot of repository state by: + +1. Identifying a target bookmark (typically an important branch like `main`) +2. Deriving required data types for commits at that bookmark +3. Collecting filenodes (Mercurial file metadata including paths, parent relationships, and linknodes) +4. Serializing this data into a compact Thrift representation +5. Storing the snapshot either in the blobstore or to a local filesystem path + +The snapshot format is defined in `features/microwave/if/microwave.thrift` and includes: + +- **Filenodes** - File metadata needed for Mercurial protocol operations +- **Changesets** - Commit metadata (currently unused but reserved for future expansion) + +Snapshots are versioned using a code version constant, allowing the format to evolve while maintaining backward compatibility. + +### Snapshot Storage + +Snapshots can be stored in two locations: + +**Blobstore** - Snapshots are written to the mutable repository blobstore using a versioned key (`microwave_snapshot_v{version}`). This is the primary storage location for production deployments. + +**Local filesystem** - Snapshots can be written to a shared filesystem path for development and testing scenarios. + +The storage location is specified when running the builder binary. + +### Cache Preloading + +When a server starts or when cache warming is triggered, microwave loads the stored snapshot and populates caches: + +1. Fetch the snapshot from blobstore or filesystem +2. Deserialize the Thrift representation +3. Convert filenodes back to internal format +4. Prime the filenodes cache with the deserialized data +5. Continue with regular cache warmup operations + +This preloading happens before the server begins serving client requests, reducing the number of cache misses during initial operation. + +## What Gets Warmed + +Microwave focuses on specific types of data that are frequently accessed and expensive to derive: + +### Filenodes + +The primary data type warmed by microwave is filenodes—Mercurial file metadata that maps file paths and filenode hashes to their parent relationships and linknode information. Filenodes are required for: + +- Mercurial wire protocol operations +- File history queries +- Mercurial manifest traversal + +Populating the filenodes cache avoids expensive database queries during initial requests. + +### Derived Data + +While microwave itself focuses on filenodes, it operates as part of the broader cache warmup process that also: + +- Derives Mercurial changesets (Bonsai to Mercurial conversion) +- Computes file history (FilenodesOnlyPublic derived data type) +- Loads manifests for the target bookmark +- Warms blobstore caches with frequently accessed blobs + +The complete cache warmup process integrates microwave preloading with manifest warming, commit graph segment loading, and derived data computation. + +### Blobstore Entries + +The cache warmup process that microwave participates in also loads manifest entries from the blobstore, populating both cachelib (in-memory cache) and memcache (shared cache) with: + +- Manifest tree nodes +- Directory metadata +- Commit graph segments + +This reduces the number of blobstore fetches needed during initial operation. + +## When Warming Happens + +Cache warming occurs in specific scenarios determined by configuration and operational needs. + +### Server Startup + +Mononoke servers can be configured to perform cache warmup at startup. The configuration is specified per-repository in the repository configuration file using `CacheWarmupParams`: + +- **bookmark** - The bookmark to warm cache for (typically `main` or another important branch) +- **commit_limit** - Maximum number of ancestor commits to warm +- **microwave_preload** - Whether to load microwave snapshot before other warmup operations + +When `microwave_preload` is enabled, the server loads the snapshot before performing other cache warming activities. + +### Builder Execution + +The microwave builder runs as a scheduled job to create updated snapshots. This job: + +- Runs periodically (frequency determined by operational needs) +- Identifies the latest derived state for configured bookmarks +- Creates new snapshots reflecting current repository state +- Overwrites previous snapshots in storage + +The builder uses the same cache warmup infrastructure to ensure derived data is available before creating the snapshot. + +### On-Demand Warming + +Cache warming can also be triggered programmatically through the `cache_warmup` function in `features/cache_warmup/src/lib.rs`. This is used by: + +- Administrative tools +- Testing infrastructure +- Server initialization sequences + +## Integration with Cache Warmup + +Microwave is one component of Mononoke's multi-phase cache warming strategy: + +### Warmup Sequence + +When cache warmup runs with microwave enabled: + +1. **Microwave preload** - Load snapshot and prime filenodes cache +2. **Blobstore and filenodes warmup** - Derive filenodes for the target bookmark, fetch all manifest entries and their linknodes +3. **Commit graph warmup** - Load commit graph segments for ancestry queries + +These phases run concurrently where possible to reduce total warmup time. + +### Warmup Kinds + +The cache warmup system distinguishes between two scenarios: + +**MicrowaveBuilder** - Used when creating snapshots. Speed is less critical as this is an offline job. All necessary derived data is computed. + +**MononokeServer** - Used during server startup. Optimized for speed to minimize the time before the server can handle traffic. Some optimizations (like skipping blobstore warmup) may be enabled via configuration. + +### Configuration + +Cache warmup configuration is specified in repository configuration files using `CacheWarmupParams`. The warmup runs at server startup for repositories assigned to each server instance through repository sharding. + +Warmup can be disabled by omitting the `cache_warmup` configuration section, leaving caches cold at startup. + +## Operational Characteristics + +Several characteristics of microwave affect its operational behavior: + +**Snapshot Staleness** - Snapshots reflect repository state at the time of builder execution. If the configured bookmark moves forward significantly between builder runs, the snapshot becomes less effective as it does not include recent commits. + +**Builder Coupling** - The builder must run periodically to maintain useful snapshots. If the builder fails or does not run, snapshots become stale and eventually may reference derived data that is no longer available. + +**Storage Overhead** - Snapshots consume blobstore space. The size depends on the number of files in manifests being snapshotted and the depth of history captured. + +**Warmup Time** - Loading a snapshot is faster than deriving data from scratch, but still takes measurable time during server startup. The warmup process must complete before the server is ready to serve traffic. + +**Versioning** - Snapshot format changes require updating the `CODEVER` constant in the Thrift definition. Old snapshots become invalid when the version changes, requiring the builder to run with the new version before servers can use updated snapshots. + +## Related Components + +Microwave interacts with several other Mononoke components: + +**Filenodes** (`filenodes/` and `repo_attributes/filenodes/`) - The primary data structure that microwave preloads. The filenodes implementation provides a `prime_cache` method specifically for microwave. + +**Cache Warmup** (`features/cache_warmup/`) - The orchestration layer that coordinates microwave with other warmup activities. + +**Warm Bookmarks Cache** (`repo_attributes/bookmarks/warm_bookmarks_cache/`) - A complementary caching mechanism that maintains up-to-date derived data for important bookmarks through continuous background derivation. + +**Derived Data** - Microwave depends on derived data being available for the target bookmark. The `FilenodesOnlyPublic` derived data type is computed during the builder process. + +**Mutable Blobstore** (`repo_attributes/mutable_blobstore/`) - Used to store snapshots in the blobstore when not using local filesystem storage. + +## Implementation Details + +Key implementation files: + +- `features/microwave/src/lib.rs` - Core preloading logic +- `features/microwave/builder/main.rs` - Builder binary +- `features/microwave/if/microwave.thrift` - Snapshot data format +- `features/cache_warmup/src/lib.rs` - Integration point for cache warmup + +The microwave builder is a standard Mononoke binary using the `mononoke_app` framework. It can be built and run using: + +```bash +buck2 build @mode/opt fbcode//eden/mononoke/features/microwave/builder:builder +buck2 run @mode/opt fbcode//eden/mononoke/features/microwave/builder:builder -- blobstore +``` + +The builder supports two subcommands: `local-path` (for filesystem storage) and `blobstore` (for blobstore storage). + +## Related Documentation + +* [Architecture Overview](1.3-architecture-overview.md) - Caching in the overall architecture +* [Derived Data](2.3-derived-data.md) - Derived data types and derivation +* [Storage Architecture](2.4-storage-architecture.md) - Detailed caching strategy +* [Servers and Services](3.1-servers-and-services.md) - Server startup and cache warmup + +Component-specific implementation details are located in `features/microwave/` and `features/cache_warmup/`. diff --git a/eden/mononoke/docs/5.1-git-support.md b/eden/mononoke/docs/5.1-git-support.md new file mode 100644 index 0000000000000..2ad0a6364838f --- /dev/null +++ b/eden/mononoke/docs/5.1-git-support.md @@ -0,0 +1,285 @@ +# Git Support + +This document explains how Mononoke supports Git clients and repositories. Git support allows standard Git clients to interact with Mononoke-backed repositories through the Git protocol while maintaining Bonsai as the canonical internal representation. + +## Overview + +Mononoke provides Git protocol support through several components: + +- **Git server** - HTTP-based server implementing the Git smart protocol +- **Format conversion** - Bidirectional mapping between Bonsai changesets and Git commits +- **Git-specific derived data** - Pre-computed Git trees and delta manifests +- **Reference management** - Git symbolic refs, tags, and branch handling +- **LFS server** - Large file storage compatible with Git LFS protocol +- **Source of truth tracking** - Migration support for repositories transitioning between VCS systems + +This integration enables Git clients to clone, fetch, and push to Mononoke repositories using standard Git operations, while Mononoke maintains a single canonical representation in Bonsai format. + +## Git Protocol Server + +The Git server (`git_server/`) implements the Git smart HTTP protocol, allowing standard Git clients to interact with Mononoke repositories. + +### Protocol Implementation + +The server handles Git protocol operations through several layers: + +**Protocol parsing** (`git/protocol/`) - Implements Git's pack protocol, including negotiation, packfile generation, and reference advertisement. + +**Packfile handling** (`git/packfile/`) - Generates and parses Git packfiles containing commits, trees, and blobs. Packfiles are the primary data transfer format in the Git protocol. + +**Packet-line format** (`git/packetline/`) - Implements Git's packet-line protocol used for framing data in Git communication. + +### Supported Operations + +The Git server handles standard Git operations: + +**Clone** - Advertises available references and generates a packfile containing the requested commits, trees, and blobs. The server converts Bonsai changesets to Git commits on-demand during packfile generation. + +**Fetch** - Determines which objects the client needs based on negotiation and generates an incremental packfile. Uses commit graph traversal to identify objects not present on the client. + +**Push** - Receives a packfile from the client, extracts Git objects, converts Git commits to Bonsai changesets, and updates references after validation and hook execution. + +**ls-remote** - Lists available references (branches and tags) with their current commit SHAs. + +### Request Routing + +The Git server uses Gotham-based HTTP routing (`git_server/src/service/`) to handle protocol endpoints. Upload-pack requests (fetch/clone) and receive-pack requests (push) are routed to separate handlers that manage the respective protocol flows. + +## Bonsai-Git Conversion + +Mononoke maintains a bidirectional mapping between Bonsai changesets and Git commits, enabling conversion in both directions. + +### Git Commits as Derived Data + +Git commits are implemented as a derived data type (`MappedGitCommitId`) in `git/git_types/src/derive_commit.rs`. When a Bonsai changeset needs to be served to Git clients, the corresponding Git commit is derived. + +The derivation process: + +1. Converts Bonsai author and date to Git signature format +2. Handles committer information (Git always requires a committer) +3. Translates file changes to Git tree structure +4. Generates Git commit object with appropriate parent references +5. Computes Git SHA-1 hash +6. Stores the commit object in the blobstore +7. Records the Bonsai ↔ Git SHA mapping + +### Metadata Conversion + +Bonsai changeset metadata maps to Git commit fields: + +**Author and committer** - Bonsai author and optional committer are converted to Git signature format (name, email, timestamp with timezone). + +**Commit message** - Preserved directly. Extra metadata fields specific to other VCS systems are omitted. + +**Parents** - Parent Bonsai changesets are mapped to their corresponding Git commits through the mapping table. + +### File Changes to Tree Objects + +Bonsai's flat list of file changes is converted to Git's hierarchical tree structure. Each directory level becomes a Git tree object referencing either subtrees (directories) or blobs (files). + +**File modes:** +- Regular files → mode 100644 +- Executable files → mode 100755 +- Symlinks → mode 120000 + +**Content addressing** - File contents are shared between Bonsai and Git representations. The Git blob hash is stored in Bonsai's content metadata. + +**Copy/move information** - Git's object model does not represent copy or rename metadata (Git infers these during diff operations). This information is not included in Git commits but is preserved in the Bonsai representation. + +### Mapping Storage + +The `BonsaiGitMapping` facet (`repo_attributes/bonsai_git_mapping/`) maintains the bidirectional mapping between Bonsai changeset IDs and Git commit SHAs. This mapping is stored in a SQL table and cached for performance. + +The mapping is populated during: +- Git import (when existing Git repositories are imported into Mononoke) +- Git push (when new Git commits are converted to Bonsai) +- Git commit derivation (when Bonsai changesets are converted to Git) + +Additional mappings are provided by: +- `BonsaiTagMapping` - Maps Git annotated tag objects to Bonsai +- `BonsaiBlobMapping` - Maps Git blob objects to file content IDs + +### Import and Export + +**gitimport** (`git/gitimport/`) - Imports existing Git repositories into Mononoke by converting Git commits to Bonsai changesets. Preserves commit graph structure and reference positions. + +**gitexport** (`git/gitexport/`) - Exports Mononoke repositories in Git format. Used for migration scenarios and backup. + +**import_tools** (`git/import_tools/`) - Utilities supporting import operations, including validation and metadata handling. + +## Git-Specific Derived Data + +Beyond Git commits, Mononoke derives additional Git-specific data to optimize protocol operations. + +### Git Trees + +Git tree objects (`GitTreeId`) represent directory structures. These are derived alongside Git commits, creating the hierarchical tree structure from Bonsai's flat file change representation. + +### Delta Manifests + +Delta manifests optimize Git packfile generation by pre-computing object delta information. Generating deltas on-demand for every fetch would be prohibitively expensive for large repositories. + +**GitDeltaManifestV2** (`git/git_types/src/delta_manifest_v2.rs`) - Tracks potential delta bases for each object in a commit, reducing packfile generation time. + +**GitDeltaManifestV3** (`git/git_types/src/delta_manifest_v3.rs`) - Enhanced version with chunking for better performance. + +**CompactedGitDeltaManifest** (`git/git_types/src/compacted_delta_manifest.rs`) - For repositories with many small commits, fetching individual delta manifests per commit creates overhead. The compacted format aggregates delta information across multiple commits, reducing the number of blobstore fetches during packfile generation. + +These derived data types are computed asynchronously off the critical write path, similar to other derived data in Mononoke. + +### Bundle URI Support + +Git's bundle URI protocol allows servers to provide pre-generated repository bundles for efficient full clones. Mononoke can pre-compute bundles (`git/bundle_uri/`) and serve them to clients, avoiding the cost of generating packfiles for complete repository clones. + +## Git Reference Management + +Git uses references (refs) to track branches, tags, and other named pointers. Mononoke provides several facets to manage Git reference semantics. + +### Symbolic References + +The `GitSymbolicRefs` facet (`repo_attributes/git_symbolic_refs/`) handles Git symbolic references, which are references that point to other references rather than directly to commits. + +The most common symbolic reference is `HEAD`, which typically points to the current branch (e.g., `HEAD → refs/heads/main`). The facet stores the symbolic reference name, target reference name, and target type (branch or tag). + +### Reference Content Mapping + +The `GitRefContentMapping` facet (`repo_attributes/git_ref_content_mapping/`) maps Git references to their content. This is used for annotated tags and other reference types that contain content beyond a simple commit pointer. + +### Bookmarks Integration + +Git branches are mapped to Mononoke bookmarks. When a Git client pushes to a branch, the corresponding bookmark is updated. When advertising references, the server queries bookmarks and formats them as Git refs. + +Reference paths follow Git conventions: +- Branches: `refs/heads/` +- Tags: `refs/tags/` +- `HEAD`: Symbolic reference to current branch + +## Git LFS Integration + +The LFS server (`lfs_server/`) implements the Git LFS protocol for large file storage. Git LFS replaces large files in Git repositories with small pointer files, storing the actual content separately. + +### LFS Protocol + +The server implements the Git LFS batch API over HTTP: + +**Batch endpoint** - Clients request upload or download URLs for multiple objects. The server returns signed URLs or direct endpoints for transferring content. + +**Upload/download endpoints** - Handle actual file content transfer. Upload accepts file content via HTTP PUT. Download serves file content via HTTP GET. + +### LFS Pointer Interpretation + +Mononoke can optionally interpret LFS pointer files. When enabled: + +1. During Git push, pointer files are detected by pattern matching +2. Both the pointer and full file contents are uploaded to the blobstore +3. The full file contents are stored in the Bonsai changeset +4. A `git_lfs` field in the Bonsai `FileChange` indicates the file should be represented as an LFS pointer in Git format + +This approach stores actual file contents in Bonsai rather than pointers, providing: +- Consistent file access across different APIs (SCS, Sapling) +- Accurate repository size statistics +- Data integrity checks covering actual content +- Proper handling in cross-repository sync operations + +When serving Git clients, files marked with the `git_lfs` field are converted back to pointer format. + +### LFS Storage + +LFS file contents are stored in the blobstore using the `Filestore` facet, which handles chunking for large files. The LFS server uses content-addressed storage, with object IDs derived from SHA-256 hashes of file contents. + +The LFS server can be configured to use an upstream LFS server for federation, allowing Mononoke to act as a cache or proxy. + +## Source of Truth Tracking + +The `GitSourceOfTruthConfig` facet (`repo_attributes/git_source_of_truth/`) tracks which VCS format is authoritative for a repository. This is used during migrations when repositories transition from external Git systems to Mononoke or vice versa. + +The facet records: +- Repository ID and name +- Source of truth designation (Mononoke or external Git) +- Migration status and mutation ID for tracking updates + +This information ensures consistency when a repository is being served through multiple VCS protocols during migration periods. + +## Request Flow Examples + +### Git Clone + +1. Client sends HTTP GET to git-upload-pack endpoint +2. Git server advertises available references (branches, tags) +3. Client sends want/have negotiation +4. Server determines which commits and objects the client needs +5. Server derives Git commits from Bonsai changesets +6. Server derives Git trees from file changes +7. Server uses delta manifests to determine object deltas +8. Server generates packfile with commits, trees, and blobs +9. Packfile is streamed to client +10. Client unpacks objects and checks out working directory + +### Git Push + +1. Client sends HTTP POST to git-receive-pack endpoint with packfile +2. Git server parses packfile and extracts Git objects +3. Server converts Git commits to Bonsai changesets +4. Server validates commits and runs hooks (via `HookManager` facet) +5. Server stores Bonsai changesets in blobstore +6. Server updates Bonsai-Git mapping in metadata database +7. Server updates commit graph index +8. Server updates bookmark (Git ref) position +9. Server confirms push success to client +10. Derived data computation is queued asynchronously + +### Git Fetch + +1. Client sends HTTP POST to git-upload-pack with have/want negotiation +2. Server uses commit graph to identify missing commits +3. Server generates incremental packfile with new objects +4. Server uses delta manifests to optimize delta computation +5. Packfile is streamed to client +6. Client applies updates to local repository + +## Component Organization + +Git support is implemented across multiple directories: + +**Protocol and server:** +- `git_server/` - Git server binary and HTTP handlers +- `git/protocol/` - Git protocol implementation +- `git/packfile/` - Packfile generation and parsing +- `git/packetline/` - Packet-line protocol format + +**Data types and conversion:** +- `git/git_types/` - Git object types, commit derivation, delta manifests +- `git/git_types/src/tree.rs` - Git tree derivation from Bonsai +- `git/git_types/src/derive_commit.rs` - Git commit derivation +- `git/git_types/src/delta_manifest_*.rs` - Delta manifest types + +**Facets (in `repo_attributes/`):** +- `bonsai_git_mapping/` - Bonsai ↔ Git SHA mapping +- `git_symbolic_refs/` - Symbolic reference handling +- `git_ref_content_mapping/` - Reference content mapping +- `git_source_of_truth/` - Source of truth configuration +- `bonsai_tag_mapping/` - Git annotated tag mapping +- `bonsai_blob_mapping/` - Git blob mapping + +**LFS:** +- `lfs_server/` - LFS server binary and protocol implementation +- `git/git_types/src/git_lfs.rs` - LFS pointer handling + +**Import/export:** +- `git/gitimport/` - Git repository import +- `git/gitexport/` - Git repository export +- `git/import_tools/` - Import utilities +- `git/import_direct/` - Direct import operations + +**Supporting components:** +- `git/bundle_uri/` - Bundle URI protocol support +- `git/check_git_wc/` - Working copy validation +- `git/git_env/` - Git environment utilities + +## Related Documentation + +- [Servers and Services](3.1-servers-and-services.md) - Overview of Git server in the service architecture +- [Repository Facets](2.2-repository-facets.md) - Git-specific facets and their usage +- [Derived Data](2.3-derived-data.md) - How Git commits and trees are derived +- [Bonsai Data Model](2.1-bonsai-data-model.md) - The canonical format from which Git objects are derived +- [Mercurial/Sapling Support](5.2-mercurial-sapling-support.md) - Comparison with Mercurial integration diff --git a/eden/mononoke/docs/5.2-mercurial-sapling-support.md b/eden/mononoke/docs/5.2-mercurial-sapling-support.md new file mode 100644 index 0000000000000..cff838fa51cb0 --- /dev/null +++ b/eden/mononoke/docs/5.2-mercurial-sapling-support.md @@ -0,0 +1,385 @@ +# Mercurial and Sapling Support + +This document explains how Mononoke supports Mercurial and Sapling clients. Mononoke was originally designed as a scalable Mercurial backend and retains extensive support for Mercurial protocols and data formats. As Mercurial evolved into Sapling, Mononoke evolved alongside it, and now primarily only support Sapling clients. + +## Historical Context + +Mononoke originated as a scalable backend for Mercurial, which served as Meta's primary version control system. The initial design focused on supporting Mercurial clients while introducing Bonsai as an internal canonical format. This approach allowed Mononoke to scale beyond traditional Mercurial server implementations while preserving client compatibility. + +As Mercurial development transitioned to the Sapling project, Mononoke continued serving clients through protocol evolution rather than architectural changes. The server maintained support for legacy Mercurial wire protocols while adding support for EdenAPI (now called SLAPI), an HTTP-based protocol that replaced older SSH-based communication. + +Today, Mononoke serves primarily Sapling clients, and Mercurial wireproto support is being removed. For clarity, this document uses "Mercurial" to refer to the version control system and its data model, and "Sapling" to refer to the current client implementation. In the future we may evolve the data model as well, in which case we will likely refer to the new data model as the "Sapling" data model. + +## Mercurial Data Model + +Mercurial uses a content-addressed data model based on nodes. Each node—whether a changeset, manifest, or file—is identified by a hash computed from its parents and content. This section describes the key Mercurial types that Mononoke handles. + +### Core Mercurial Types + +Mercurial types are defined in `mercurial/types/`. The primary identifier types are: + +**HgChangesetId** (`mercurial/types/src/nodehash.rs`) +- Identifies a Mercurial changeset (commit) +- SHA-1 hash computed from parent hashes and changeset content +- 20-byte identifier, typically displayed in hexadecimal + +**HgManifestId** (`mercurial/types/src/nodehash.rs`) +- Identifies a Mercurial manifest (directory tree) +- Computed from parent manifest hashes and directory contents +- Forms a DAG independent of changesets +- Note that Mononoke only supports tree manifests (not traditional Mercurial flat manifests) + +**HgFileNodeId** (`mercurial/types/src/nodehash.rs`) +- Identifies a file version (filenode) +- Computed from parent filenode hashes and file content +- Each file has its own history DAG + +### Node Computation + +Mercurial node hashes follow a consistent pattern: + +``` +node_hash = SHA1(p1_hash || p2_hash || content) +``` + +Where: +- `p1_hash` is the first parent (null hash if no parent) +- `p2_hash` is the second parent (null hash if fewer than two parents) +- `content` is the serialized content specific to the node type + +This computation creates content-addressed Merkle DAGs for changesets, manifests, and files. The same content with different parents produces different hashes, embedding history in the identity. + +### Mercurial Structures + +**Changesets** (`mercurial/types/src/blobs/changeset/`) +- Contain commit metadata (author, date, message) +- Reference a manifest ID (the root directory) +- List parent changeset IDs (0-2 parents) +- Include extra metadata (key-value pairs) + +**Manifests** (`mercurial/types/src/blobs/manifest.rs`) +- Map filenames to file IDs and metadata +- Represent directory contents +- Can reference sub-manifests for subdirectories +- Stored as sorted lists of entries + +**Filenodes** (`mercurial/types/src/blobs/file.rs`) +- Represent specific file versions +- Store file content or deltas +- Include metadata (flags, copy information) +- Form per-file history DAGs + +### Wire Formats + +Mercurial wire protocols use several serialization formats: + +**Revlog Format** (`mercurial/revlog/`) +- On-disk storage format used by Mercurial +- Contains compressed deltas and indexes +- Mononoke can read revlog data for imports + +**Bundle2 Format** (`mercurial/bundles/`) +- Wire protocol format for exchanging data +- Streams of parts containing changesets, manifests, files +- Supports capabilities negotiation +- Used for push and pull operations + +**Wirepack Format** (`mercurial/bundles/src/wirepack/`) +- Alternative wire format for file data +- More efficient than changegroups for large files +- Used by remotefilelog extension + +## Bonsai ↔ Mercurial Conversion + +Mononoke stores commits internally as Bonsai changesets and converts to Mercurial format when serving clients. The `BonsaiHgMapping` facet maintains bidirectional mappings between these representations. + +### BonsaiHgMapping Facet + +The `BonsaiHgMapping` facet (`repo_attributes/bonsai_hg_mapping/`) provides mapping operations: + +**Core Operations:** +- `get_hg_from_bonsai` - Convert Bonsai changeset ID to Mercurial changeset ID +- `get_bonsai_from_hg` - Convert Mercurial changeset ID to Bonsai changeset ID +- `add` - Store a new mapping entry +- `get` - Batch lookup supporting both directions + +**Storage:** +- Mappings stored in metadata database +- Cached for performance (via `CachingBonsaiHgMapping`) +- Written during commit ingestion or derivation + +### Conversion Process + +When a Mercurial client pushes commits: + +1. **Receive Mercurial data** - Server receives changesets in Mercurial format via wire protocol +2. **Convert to Bonsai** - Transform Mercurial changeset structure to Bonsai format +3. **Store core data** - Write Bonsai changeset and file contents to blobstore +4. **Create mapping** - Store Bonsai ↔ Mercurial changeset ID mapping +5. **Store Mercurial format** - Optionally preserve original Mercurial bytes + +When a client requests data: + +1. **Receive request** - Client requests by Mercurial changeset ID +2. **Resolve to Bonsai** - Look up corresponding Bonsai changeset ID +3. **Operate on Bonsai** - Perform operations using Bonsai representation +4. **Derive Mercurial data** - Generate Mercurial changeset if needed +5. **Return Mercurial format** - Convert response to Mercurial wire format + +### Conversion Characteristics + +The conversion between Bonsai and Mercurial has the following characteristics: + +**Not Bijective** - Multiple Bonsai changesets can map to the same Mercurial changeset ID if they differ only in metadata not represented in Mercurial format. In practice, mappings are effectively one-to-one due to compatibility constraints. + +**Deterministic** - Given a Bonsai changeset, the derived Mercurial changeset ID is deterministic. This allows regeneration and verification. + +**Async Derivation** - Mercurial representations can be derived asynchronously after Bonsai commits are stored. See the Derived Data section below. + +## Mercurial-Specific Derived Data + +Several derived data types exist specifically to support Mercurial and Sapling clients. These types provide data in formats expected by Mercurial protocols. + +### Mapped Mercurial Changeset + +**MappedHgChangesetId** (`derived_data/mercurial_derivation/`) + +This derived data type generates a Mercurial changeset from a Bonsai changeset. The derivation: + +1. Converts Bonsai file changes to Mercurial manifest format +2. Derives the manifest tree (see below) +3. Constructs Mercurial changeset bytes with metadata +4. Computes the Mercurial changeset ID +5. Stores the mapping in `BonsaiHgMapping` + +This type is required for serving Mercurial clients and is typically derived on-demand or by background workers. + +### Mercurial Augmented Manifests + +**RootHgAugmentedManifestId** (`derived_data/mercurial_derivation/`) + +Augmented manifests are content-addressed directory trees in Mercurial format with additional metadata. Unlike traditional Mercurial manifests, augmented manifests include: + +- File sizes +- Content hashes (SHA-1 and SHA-256) +- File type information + +Augmented manifests enable efficient serving of tree data to Sapling clients without requiring full manifest computation. The derivation constructs Mercurial-compatible tree structures from Bonsai file changes. + +Augmented manifests are stored in the blobstore using sharded storage for large directories. See `mercurial/types/src/sharded_augmented_manifest.rs` for the data structure. + +### Filenodes + +**Filenodes** (`derived_data/filenodes_derivation/`) + +Filenodes represent Mercurial's per-file history tracking. Each filenode records: + +- File path +- File node ID (Mercurial file hash) +- Changeset ID (linkrev - the changeset that introduced this version) +- Parent file node IDs + +Filenodes are required for Mercurial wire protocol compatibility, particularly for operations like `getfiles` and `getpackv1`. The filenode derivation: + +1. Examines file changes in the Bonsai changeset +2. Computes Mercurial file node IDs based on content and parents +3. Records the linkrev (changeset containing the file change) +4. Stores filenode information in the metadata database + +Legacy filenode implementation is in `repo_attributes/filenodes/`, while the derived data implementation is in `derived_data/filenodes_derivation/`. + +### Derivation Dependencies + +Mercurial-specific derived data types have dependencies: + +- **MappedHgChangesetId** depends on **Fsnodes** (for manifest structure) +- **RootHgAugmentedManifestId** depends on **Fsnodes** +- **Filenodes** depend on **MappedHgChangesetId** (for linkrevs) + +These dependencies ensure that Mercurial data can be derived consistently from Bonsai commits. + +## Protocol Support + +Mononoke supports multiple protocols for Mercurial and Sapling clients, reflecting the evolution of client-server communication. + +### EdenAPI (SLAPI) + +EdenAPI, now called SLAPI (Sapling Remote API), is an HTTP-based protocol that replaced older Mercurial wire protocols. The implementation is in `edenapi_service/`. + +**Protocol Characteristics:** +- HTTP/2 based +- Structured requests and responses (CBOR or JSON serialization) +- Streaming support for large responses +- Authentication via headers +- Supports concurrent requests + +**Major Endpoints** (`edenapi_service/src/handlers/`): + +- **Files** (`files.rs`) - Fetch file contents by content hash or path +- **Trees** (`trees.rs`) - Fetch directory manifests +- **Commit** (`commit.rs`) - Upload commits and push operations +- **History** (`history.rs`) - Fetch file or directory history +- **Bookmarks** (`bookmarks.rs`) - Fetch bookmark values +- **Lookup** (`lookup.rs`) - Resolve commit hashes and bookmarks +- **Blame** (`blame.rs`) - Fetch blame annotations +- **Land** (`land.rs`) - Landing (merge) operations +- **Commit Cloud** (`commit_cloud.rs`) - Sync uncommitted work across machines + +**Serving Process:** + +1. Client sends HTTP request to specific endpoint +2. Middleware authenticates and extracts metadata +3. Handler deserializes request +4. Handler accesses repository via `mononoke_api` layer +5. Repository operations use facets to fetch Bonsai data +6. Derived data is generated if needed (Mercurial changesets, manifests) +7. Response is serialized and streamed to client + +EdenAPI/SLAPI is the primary protocol used by modern Sapling clients and EdenFS. + +### Legacy Wire Protocol + +The legacy Mercurial wire protocol (`wireproto_handler/`) supports older Mercurial clients using SSH-based communication. This protocol: + +- Uses bundle2 format for data exchange +- Supports capabilities negotiation +- Handles pushes via changegroup bundles +- Provides pull operations via getbundle command + +The wire protocol handler converts between bundle2 format and Mononoke's internal operations. While still supported, most clients have migrated to EdenAPI/SLAPI. + +### Remotefilelog + +Remotefilelog is a Mercurial extension that avoids downloading full file history by fetching file contents on-demand. Mononoke supports remotefilelog through: + +**Getpack Endpoint** - Fetches file contents in wirepack format +**Getcommitdata** - Fetches commit metadata without manifests +**Gettreepack** - Fetches tree manifests without file contents + +Remotefilelog support is integrated into both the wire protocol handler and EdenAPI. File data is served from the blobstore, with Mercurial file node IDs resolved via filenodes derived data. + +## Bundle Formats + +Mercurial uses bundle formats to transfer data between client and server. Mononoke supports reading and writing these formats for protocol compatibility. + +### Bundle2 + +Bundle2 (`mercurial/bundles/`) is the primary wire protocol format. A bundle2 stream consists of: + +**Header** - Protocol version and capabilities +**Parts** - Typed chunks of data (changesets, manifests, files) +**Parameters** - Metadata about each part + +**Common Parts:** + +- **Changegroup** - Contains changesets, manifests, and file deltas +- **Obsmarkers** - Obsolescence markers (deprecated commits) +- **Pushkey** - Bookmark updates +- **Reply parts** - Server responses to client requests + +Bundle2 parsing (`mercurial/bundles/src/bundle2_encode.rs`) handles: +- Stream decompression +- Part deserialization +- Capability negotiation +- Error handling + +### Changegroup Format + +Changegroups (`mercurial/bundles/src/changegroup/`) are sequences of deltas for changesets, manifests, and files. Each entry contains: + +- Node ID +- Parent node IDs +- Delta or full content +- Linkrev (for files and manifests) + +Changegroup unpacking (`unpacker.rs`) reconstructs full content from deltas, while packing (`packer.rs`) generates efficient delta sequences for transmission. + +### Wirepack Format + +Wirepack (`mercurial/bundles/src/wirepack/`) is an alternative to changegroups used primarily for file data. Wirepack format: + +- Streams file content with metadata +- Avoids delta chains +- More efficient for large files +- Used by remotefilelog and EdenAPI + +## Migration and Compatibility + +Mononoke maintains compatibility with both legacy Mercurial clients and modern Sapling clients through careful protocol support and data format handling. + +### Protocol Version Support + +Mononoke servers negotiate capabilities with clients: + +- **Legacy clients** - Use bundle2 wire protocol over SSH +- **Remotefilelog clients** - Use getpack endpoints over HTTP +- **Sapling clients** - Use EdenAPI/SLAPI over HTTP + +Capability negotiation allows servers to advertise supported features while clients can fall back to compatible operation modes. + +### Client Type Detection + +The server identifies client type through: + +- User-agent headers (for HTTP requests) +- Capability strings (for wire protocol) +- Requested endpoints (EdenAPI vs legacy commands) + +This allows serving different clients from the same server infrastructure. + +### Data Format Compatibility + +Mononoke ensures Mercurial data format compatibility by: + +**Preserving Hashes** - Derived Mercurial changeset IDs match what a native Mercurial server would produce for the same commits. This ensures clients can verify data integrity. + +**Supporting Legacy Formats** - Bundle2, changegroup, and wirepack formats are fully supported for clients that require them. + +**Maintaining Mapping Consistency** - The `BonsaiHgMapping` provides stable Bonsai ↔ Mercurial ID mappings across server restarts and derivation operations. + +## Evolution from Mercurial to Sapling + +The transition from Mercurial to Sapling occurred gradually, with Mononoke adapting to serve both client types: + +**Protocol Migration** - The shift from SSH-based wire protocol to HTTP-based EdenAPI improved performance and simplified deployment. Mononoke added EdenAPI support while maintaining wire protocol compatibility. + +**Client Rebranding** - As Mercurial clients were rebranded to Sapling, the server continued serving the same protocols with updated client-agent strings. + +**Feature Additions** - New features like Commit Cloud and suffix query were added to EdenAPI without breaking compatibility with older clients. + +**Deprecation Path** - Legacy wire protocol support remains for compatibility but is not actively developed. New features are added to SLAPI. + +## Implementation Locations + +Mercurial and Sapling support is distributed across several directories: + +**Core Types and Formats:** +- `mercurial/types/` - Mercurial type definitions +- `mercurial/bundles/` - Bundle2 and wire format handling +- `mercurial/revlog/` - Revlog format support +- `mercurial/mutation/` - Mutation (history editing) tracking + +**Mappings:** +- `repo_attributes/bonsai_hg_mapping/` - Bonsai ↔ Mercurial mapping facet + +**Derived Data:** +- `derived_data/mercurial_derivation/` - Mercurial changeset and manifest derivation +- `derived_data/filenodes_derivation/` - Filenode derivation +- `repo_attributes/filenodes/` - Legacy filenode storage + +**Protocols:** +- `edenapi_service/` - EdenAPI/SLAPI HTTP server +- `wireproto_handler/` - Legacy wire protocol handler + +**Server:** +- `server/` - Main Mononoke server (serves SLAPI) + +## Related Documentation + +* [Architecture Overview](1.3-architecture-overview.md) - How Mercurial support fits into Mononoke's architecture +* [Bonsai Data Model](2.1-bonsai-data-model.md) - The canonical format that Mercurial data converts to +* [Repository Facets](2.2-repository-facets.md) - BonsaiHgMapping and other facets +* [Derived Data](2.3-derived-data.md) - Framework for Mercurial-specific derived data types +* [Git Support](5.1-git-support.md) - Comparison with Git integration approach + +Component-specific documentation for Mercurial types, protocols, and derived data lives in the respective directories. diff --git a/eden/mononoke/docs/6.1-rate-limiting-and-load-shedding.md b/eden/mononoke/docs/6.1-rate-limiting-and-load-shedding.md new file mode 100644 index 0000000000000..4971dde645b4d --- /dev/null +++ b/eden/mononoke/docs/6.1-rate-limiting-and-load-shedding.md @@ -0,0 +1,330 @@ +# Rate Limiting and Load Shedding + +This document describes Mononoke's rate limiting and load shedding mechanisms, which control request load to maintain system stability at scale. + +## Introduction + +Mononoke serves source control traffic for repositories with thousands of commits per day and hundreds of thousands of clients. Without load control, individual clients or groups of clients can overwhelm system resources, degrading service for all users. Rate limiting and load shedding provide mechanisms to control this load. + +These mechanisms operate at different layers of the system: rate limiting controls individual client behavior based on request patterns, while load shedding responds to system-wide resource pressure. Together, they maintain service availability under varying load conditions. + +## Rate Limiting vs. Load Shedding + +Rate limiting and load shedding serve distinct purposes: + +### Rate Limiting + +Rate limiting restricts the number of requests from specific clients within a time window. Limits are configured based on metrics tracked per client, such as request rate or egress bandwidth. When a client exceeds its limit, additional requests are rejected with HTTP 429 (Too Many Requests) responses. + +**Characteristics:** +- Targets specific clients or groups based on identity +- Measured over configured time windows +- Applied before client behavior impacts the system +- Configuration specifies which metrics to track and threshold values + +**Example:** Reject requests from a specific client identity when that client's EdenAPI request rate exceeds 1 million requests per second globally. + +### Load Shedding + +Load shedding rejects requests when system resources reach critical thresholds. Rather than targeting individual clients, load shedding responds to resource utilization (CPU, memory, queue depth) across the service. When load shedding activates, requests are rejected based on priority, with lower-priority traffic dropped first. + +**Characteristics:** +- Responds to system-wide resource metrics +- Activates when CPU, memory, or other resources exceed thresholds +- Can prioritize traffic by dropping lower-priority requests first +- Protects system stability during overload conditions + +**Example:** When regional CPU utilization exceeds 95%, reject all incoming requests until utilization returns to acceptable levels. + +## Rate Limiting Framework + +The rate limiting framework is implemented in `rate_limiting/` and integrated into request handling middleware. + +### Tracked Metrics + +Rate limits can be configured for various metrics: + +**Request Metrics:** +- `EdenApiQps` - Requests per second to the EdenAPI service +- `LocationToHashCount` - Commit location-to-hash queries + +**Data Transfer Metrics:** +- `EgressBytes` - Bytes sent to clients +- `TotalManifests` - Manifest objects served +- `GetpackFiles` - File objects served via getpack + +**Commit Metrics:** +- `Commits` - Commits created or modified +- `CommitsPerAuthor` - Commits attributed to a specific author +- `CommitsPerUser` - Commits from a specific user identity + +Each metric is tracked over a configured time window (e.g., 1 second, 1 minute) to determine if limits are exceeded. + +### Scope Configuration + +Rate limits can apply at different scopes: + +**Global Scope** - Limits apply across all regions and server instances. A client exceeding the global limit is throttled regardless of which region or server handles the request. Global limits require coordination across server instances. + +**Regional Scope** - Limits apply within a single geographic region or data center. A client might be throttled in one region while remaining under limits in another region. Regional limits allow independent capacity management. + +### Target Specification + +Rate limits target specific clients or groups: + +**Main Client ID** - Identifies individual client applications or services. Each client is assigned a unique identifier used for quota tracking. The main client ID can include additional context such as Sandcastle job alias for CI traffic. + +**Identity Set** - Targets clients matching a set of identities. Identities include service identity (e.g., `SERVICE_IDENTITY:quicksand`), machine tier (e.g., `MACHINE_TIER:sandcastle`), or other attributes. All specified identities must be present for the rule to apply. + +**Static Slice** - Applies limits to a percentage of clients based on consistent hashing of client hostname. This allows gradual rollout of new limits by targeting an increasing slice of traffic. + +When no target is specified, the rate limit applies to all traffic. + +### Configuration Structure + +Rate limits are configured using Thrift-based configuration files that can be updated dynamically. Each rate limit specifies: + +- **Target** - Which clients the limit applies to (optional, defaults to all clients) +- **Metric** - What to measure (EdenApiQps, EgressBytes, etc.) +- **Window** - Time duration for measurement (in seconds) +- **Scope** - Global or Regional +- **Limit** - Threshold value above which requests are rejected +- **Status** - Whether the limit is enabled or disabled + +Configuration is loaded via cached config handles and updates are applied without service restart. Configuration files define arrays of rate limits, allowing multiple limits with different targets and metrics to be active simultaneously. + +### Request Flow + +Rate limiting is checked at the beginning of request processing, in the middleware layer before expensive operations are performed: + +1. **Extract client metadata** - The request handler extracts the client's main ID and identity set from connection metadata +2. **Check applicable limits** - The rate limiter identifies which configured limits apply to this client based on target matching +3. **Query metric values** - Current metric values are retrieved from in-memory counters or external metrics systems +4. **Compare to thresholds** - Each applicable limit's current value is compared to its configured threshold +5. **Accept or reject** - If any limit is exceeded, the request is rejected with HTTP 429; otherwise, processing continues +6. **Update counters** - Accepted requests update relevant metric counters for future limit checks + +This early rejection avoids consuming resources for requests that will be dropped, and the 429 response code triggers exponential backoff in Sapling clients. + +## Load Shedding Framework + +Load shedding is implemented alongside rate limiting in the `rate_limiting/` component and uses similar configuration structures. + +### Load Shedding Metrics + +Load shedding decisions are based on system resource metrics: + +**Local Counters** - Metrics exposed by the Mononoke process itself via the fb303 interface, such as CPU utilization, memory usage, request queue depth, or thread pool saturation. These are prefixed with `mononoke.` and available without external queries. + +**External Counters** - Metrics from external systems such as ODS (Operational Data Store), including storage backend latency, database connection pool utilization, or cache hit rates. External counters require periodic queries to external metrics systems, with results cached locally. + +The choice between local and external counters depends on what resource triggers load shedding. Backend storage pressure might use external counters for storage system metrics, while frontend overload uses local CPU and memory metrics. + +### Load Shedding Configuration + +Each load shed limit specifies: + +- **Target** - Which clients are shed when the limit is exceeded (optional) +- **Metric** - Resource metric to monitor (local or external counter) +- **Threshold** - Value above which load shedding activates +- **Status** - Whether the limit is enabled or disabled + +Multiple load shed limits can be configured with different targets, allowing progressive shedding. Lower-priority traffic is shed at lower thresholds, while critical traffic is only shed when resources are critically exhausted. + +**Example configuration:** +- At 90% CPU: Shed requests from `SERVICE_IDENTITY:quicksand` +- At 95% CPU: Shed all requests + +This progressive approach preserves service for high-priority traffic while protecting the system from complete failure. + +### Load Shedding Decision Process + +Load shedding is checked for each request, similar to rate limiting: + +1. **Fetch current metrics** - Resource utilization metrics are retrieved from local counters or external metrics cache +2. **Evaluate thresholds** - Each configured load shed limit is checked to determine if its threshold is exceeded +3. **Identify targets** - For exceeded thresholds, the configured target is checked against the request's client metadata +4. **Shed or allow** - If a matching load shed limit is exceeded, the request is rejected; otherwise, processing continues + +Shed requests receive the same HTTP 429 response as rate-limited requests, triggering client backoff behavior. + +## Load Limiter Service + +The Load Limiter service (`facebook/load_limiter/`) is an external microservice that coordinates load-aware job scheduling for continuous integration systems. + +### Integration with Job Scheduling + +The Load Limiter implements the Jupiter External Dependencies interface, which allows jobs to block on arbitrary external conditions. When load is high, jobs wait in queue rather than executing and failing due to service overload. + +The service implements three operations: + +**acquire** - Attempts to acquire the dependency. Returns success if load is acceptable, or blocked if load exceeds thresholds. Jobs receiving blocked responses remain in the scheduler queue rather than executing. + +**canAcquire** - Checks whether the dependency could be acquired without actually acquiring it. Used by schedulers to predict wait times. + +**release** - Releases the dependency when a job completes. For source control dependencies, this is typically a no-op since Mononoke load is not quota-based. + +### Load-Aware Scheduling + +The Load Limiter monitors Mononoke metrics (CPU utilization, request rates, error rates) and returns blocked responses when these metrics exceed configured thresholds. The relationship between Mononoke load and Load Limiter responses: + +| Avg CPU Utilization | Mononoke Response | Load Limiter Response | Job Outcome | +|---------------------|-------------------|----------------------|-------------| +| 80% | 200 (success) | Acquired | Job executes successfully | +| 97% | Intermittent 200/429 | Blocked | Job queues, executes when load decreases | +| 99% | 429 (rate limited) | Blocked | Job queues until system recovers | + +When jobs are blocked, they remain in the scheduler queue rather than consuming worker resources and failing. This reduces the number of failed CI jobs caused by transient Mononoke overload. + +### Configuration and Monitoring + +The Load Limiter uses separate configuration (`scm/mononoke/load_limiter/load_limiter`) specifying which Mononoke metrics to monitor and threshold values for blocking jobs. Configuration includes: + +- **Metrics to monitor** - ODS counters for CPU, QPS, error rates +- **Blocking thresholds** - Values above which jobs are blocked +- **Query intervals** - How frequently to poll metrics + +The service logs decisions to Scuba (`mononoke_load_limiter` dataset) for analysis of blocking patterns and load correlation. + +## Blobstore-Level Rate Limiting + +In addition to request-level rate limiting, Mononoke can throttle blobstore operations using the ThrottledBlob decorator. + +### ThrottledBlob Implementation + +ThrottledBlob (`blobstore/throttledblob/`) wraps a blobstore and applies rate limiting to individual operations. It can throttle on two dimensions: + +**Operations per Second (QPS):** +- Read QPS limit - Maximum get operations per second +- Write QPS limit - Maximum put operations per second + +**Bytes per Second:** +- Read bytes limit - Maximum bytes read per second +- Write bytes limit - Maximum bytes written per second +- Burst allowance - Temporary exceeding of limits for large blobs + +Throttling uses token bucket rate limiting (via the `governor` crate) with jitter to prevent thundering herd effects when multiple operations wait for tokens. + +### Use Cases + +Blobstore throttling is used to: + +**Protect Storage Backends** - Limit load on underlying storage systems (SQL, S3, Manifold) to prevent overwhelming them during traffic spikes or bulk operations. + +**Control Resource Consumption** - Limit network bandwidth consumption when multiple repositories share network capacity. + +**Test Resilience** - Simulate slow storage during testing by applying artificial throttling to verify timeout and retry behavior. + +Throttled blobstores appear in the blobstore decorator stack between the application and storage backend, applying limits before requests reach the underlying storage. + +## Client Behavior + +Sapling and EdenFS clients implement retry behavior for rate limiting and load shedding responses: + +**429 Responses (Rate Limit/Load Shed):** +- Trigger exponential backoff with a base of 2 seconds +- Maximum backoff of 2^3 = 8 seconds +- After maximum retries, the operation fails and reports an error to the user + +**Other Error Responses:** +- Trigger linear retries increasing by 1 second per attempt +- Different retry behavior reflects lower expected success rate for non-throttling errors + +**Data Preservation:** +- Successfully fetched data is retained locally even if subsequent operations fail +- Partial clone/fetch operations leave the repository in a usable state with whatever data was retrieved + +**EdenFS Behavior:** +- File access operations that exceed retries result in filesystem errors (ENOENT, EIO) +- Checkout operations fail if files cannot be fetched, preventing incomplete checkouts in CI environments + +This client behavior provides automatic recovery from transient overload while preventing indefinite hangs. + +## Configuration and Deployment + +Rate limiting and load shedding configuration is managed through Mononoke's repository configuration system: + +**Configuration Storage:** +- Rate limits defined in Thrift-based configuration files +- Loaded via ConfigStore with cached config handles +- Updates applied dynamically without service restart + +**Configuration Validation:** +- Configuration is validated during loading +- Invalid targets or metrics are rejected +- Deployment errors are detected before configuration takes effect + +**Monitoring Integration:** +- Rate limit decisions logged to Scuba for analysis +- ODS counters track reject rates per limit +- Dashboards show which limits are actively rejecting traffic + +Configuration updates follow standard Mononoke configuration deployment practices, with changes committed to version control and deployed through the configuration service. + +## Implementation Details + +The rate limiting implementation consists of several components: + +**RateLimiter Trait** (`rate_limiting/src/lib.rs`) +- Defines the interface for checking rate limits and load shedding +- `check_rate_limit` - Verifies if a request exceeds configured limits +- `check_load_shed` - Verifies if system load requires shedding +- `bump_load` - Updates metric counters for accepted requests +- `find_rate_limit` - Retrieves applicable limits for a client + +**RateLimitEnvironment** (`rate_limiting/src/lib.rs`) +- Holds rate limiting state: configuration, counter manager, category +- Provides factory method for creating rate limiters with appropriate configuration +- Manages ODS counter manager for external metric queries + +**Configuration Handling** (`rate_limiting/src/config.rs`) +- Deserializes Thrift configuration into internal types +- Validates target specifications and metric definitions +- Converts between configuration format and runtime structures + +**Middleware Integration** (`server/repo_listener/src/request_handler.rs`) +- Checks load shedding at request start, before expensive operations +- Logs shedding decisions to Scuba with client metadata +- Returns 429 responses for shed requests + +**Counter Management** (`rate_limiting/src/facebook.rs` for internal builds) +- Maintains in-memory counters for tracked metrics +- Periodically fetches external ODS counters +- Provides counter values for limit comparison + +The OSS build (`rate_limiting/src/oss.rs`) provides a minimal implementation that always allows requests, as full rate limiting depends on internal infrastructure. + +## Related Components + +Several Mononoke components interact with rate limiting: + +**Request Handlers** - All frontend servers (Mononoke, SCS, Git, LFS) check rate limits in their request handling middleware before processing requests. + +**Repository Facets** - The `repo_permission_checker` facet is consulted alongside rate limiting to enforce both authorization and load control. + +**Monitoring Infrastructure** - Scuba logging and ODS counters provide visibility into rate limiting decisions and load patterns. + +**Configuration System** - Cached config handles enable dynamic rate limit updates without service restart. + +## Summary + +Mononoke's rate limiting and load shedding mechanisms operate at multiple layers to control request load: + +**Request-Level Rate Limiting** - Tracks per-client metrics (QPS, egress bytes, commits) over time windows, rejecting requests from clients exceeding configured thresholds. Applies at global or regional scope. + +**System-Level Load Shedding** - Monitors resource utilization (CPU, memory, queue depth) and sheds requests when thresholds are exceeded, with progressive shedding based on request priority. + +**Blobstore Throttling** - Limits storage operations (QPS and bytes/sec) to protect backend storage systems from overload. + +**Load-Aware Job Scheduling** - The Load Limiter service integrates with CI systems to queue jobs instead of executing them during high load periods. + +These mechanisms use dynamic configuration, early request rejection, and coordinated client backoff to maintain service stability under varying load conditions. Configuration is deployed through standard configuration management with monitoring integration for observability. + +## Related Documentation + +- [Servers and Services](3.1-servers-and-services.md) - How servers integrate rate limiting +- [Architecture Overview](1.3-architecture-overview.md) - System architecture and request flow +- [Storage Architecture](2.4-storage-architecture.md) - ThrottledBlob and storage stack +- [Monitoring and Observability](6.3-monitoring-and-observability.md) - Monitoring rate limiting decisions + +Component-specific details are documented in `rate_limiting/`, `facebook/load_limiter/`, and `blobstore/throttledblob/` directories. diff --git a/eden/mononoke/docs/6.2-walker-and-validation.md b/eden/mononoke/docs/6.2-walker-and-validation.md new file mode 100644 index 0000000000000..5f202eb9892ae --- /dev/null +++ b/eden/mononoke/docs/6.2-walker-and-validation.md @@ -0,0 +1,293 @@ +# Walker and Validation + +This document explains the walker tool, which traverses Mononoke's data graph to validate integrity, verify storage durability, and perform analysis operations. + +## Overview + +The walker (`jobs/walker/`) is a graph traversal tool that represents Mononoke's data as a directed graph of nodes connected by edges. It traverses this graph to validate data consistency, verify storage durability across multiple blobstore backends, extract repository data for analysis, and measure compression characteristics. + +The walker operates as both a background job for continuous validation and a command-line tool for on-demand operations. It can process entire repositories or work on sampled subsets to handle large-scale repositories within memory constraints. + +## Graph Representation + +The walker defines a graph schema that models Mononoke's data structures as nodes with relationships represented by edges. This abstraction spans both blobstore-stored data (Bonsai changesets, file contents, manifests) and SQL-stored data (VCS mappings, bookmark data). + +**Node Types:** +- **Bonsai changesets** - Commit metadata and file change lists +- **File content** - Actual file data stored in the blobstore +- **Manifests** - Directory structures (fsnodes, unodes, skeleton manifests, Git trees) +- **VCS mappings** - Bonsai to Git/Mercurial/SVN mappings +- **Derived data** - Filenodes, blame, fastlog, and other computed indexes +- **Bookmarks** - Branch pointers and their history + +**Edge Types:** +Edges represent relationships between nodes. For example: +- Parent-child commit relationships +- Changeset-to-file-content references +- Manifest-to-file references +- File-to-changeset backlinks (creating cycles) + +The graph is discovered dynamically during traversal. The walker expands each node by loading its data and following outgoing edges to discover connected nodes. This dynamic approach handles Mononoke's large data sets without requiring the entire graph to be loaded into memory. + +### Graph Characteristics + +**Cycles:** The graph contains cycles due to backlinks from files to changesets (for example, Mercurial filenodes link back to changesets). Visit tracking prevents infinite loops during traversal. + +**Mutability:** Most data represented in the graph is immutable. Exceptions include bookmarks (which are resolved once at the start of a walk) and certain mappings that may be incomplete during derivation. The walker handles these cases by allowing revisits until data reaches a terminal state. + +**Traversal Depth:** A walk is "deep" if it covers the entire repository history by following parent relationships. A walk is "shallow" if it only examines data reachable from specific commits without traversing history. + +## Subcommands + +The walker provides four subcommands, each serving a distinct operational purpose. + +### Scrub + +The scrub subcommand verifies storage durability in multiplexed blobstore configurations. When Mononoke writes to multiple blobstore backends for redundancy, scrub checks that each component blobstore contains data for every key. + +**Operation:** +1. Traverse the graph visiting all nodes +2. For each blob accessed, verify it exists in all component blobstores +3. If a blob is missing from one backend but present in another, record the inconsistency +4. Optionally repair missing blobs by copying from available backends + +**Use Cases:** +- Detecting component blobstores with missing data +- Verifying that keys are present in all configured storage backends +- Ensuring Mononoke can operate on a single component store if necessary +- Repairing inconsistencies after storage backend issues + +Scrub integrates with the underlying `ScrubBlobstore`, which provides callbacks when issues are detected. The blobstore healer job (`jobs/blobstore_healer/`) performs similar healing operations continuously, processing the multiplexed blobstore write-ahead log. Scrub serves as a verification and on-demand repair tool. + +### Validate + +The validate subcommand checks data validity by traversing the graph and verifying references. This detects data corruption, missing data, or inconsistencies that might indicate bugs in write or derivation logic. + +**Validations Performed:** +- Verify all referenced nodes exist and are loadable +- Check linknode consistency (filenodes reference valid changesets) +- Validate phase data (public commits labeled correctly) +- Detect dangling references +- Verify data structure integrity + +**Operation:** +1. Traverse the graph from specified starting points (bookmarks, changesets) +2. Load each node's data and validate its structure +3. Verify outgoing edges point to valid, loadable nodes +4. Report validation errors with source node context + +The validate operation logs issues to Scuba with route information, allowing operators to trace back from problematic nodes to their sources in the graph. + +### Corpus + +The corpus subcommand extracts a subset of repository data and writes it to disk in a structured format. This supports offline analysis, compression experiments, and investigation of repository characteristics. + +**Operation:** +1. Traverse the graph, associating nodes with repository paths where applicable +2. Extract blob data for selected node types +3. Write blobs to disk in an organized directory structure +4. Support sampling by path hash or path regex to control corpus size + +**Directory Layout:** + +The corpus is written to a directory structure that separates node types and preserves repository paths: + +``` +/ + / + / + .mononoke,/ + / + +``` + +For example: +``` +FileContent/root/src/main.rs/.mononoke,/49/f0/c1/repo123.content.blake2.49f0... +``` + +The repository path and blob key are percent-encoded to prevent conflicts with the magic `.mononoke,` directory name (which contains a comma, and commas are percent-encoded). This encoding allows shell operations to distinguish blob structure from repository paths. + +Node types without repository paths (such as Bonsai changesets) omit the path component: +``` +BonsaiChangeset/.mononoke,/ab/cd/ef/repo123.changeset.abcdef... +``` + +**Sampling:** +The corpus subcommand supports sampling by node hash or repository path pattern. This allows extracting representative subsets of large repositories for analysis. Sampling can be based on hash ranges (`--sample-rate` and `--sample-offset`) or path regex patterns. + +### Compression Benefit + +The compression-benefit subcommand measures potential storage savings from applying compression to individual blobs. It computes the size of blobs before and after zstd compression to evaluate compression ratios. + +**Operation:** +1. Traverse the graph visiting blob nodes +2. Fetch blob data and measure uncompressed size +3. Apply zstd compression and measure compressed size +4. Report compression ratios for different blob types + +This analysis helps evaluate storage optimization strategies and understand the compression characteristics of repository data. The results inform decisions about packblob configuration and compression settings. + +## Graph Traversal Implementation + +The walker uses bounded traversal to process the graph efficiently. The `bounded_traversal_stream` function dynamically unfolds nodes in parallel, expanding edges and queueing child nodes for processing. + +### Visit Tracking + +To prevent revisiting nodes (which would cause infinite loops in cyclic graphs), the walker maintains concurrent hash maps tracking visited nodes. The `WalkStateCHashmap` stores node identities and prevents re-expansion of nodes already seen during the current walk. + +Visit tracking has memory implications. For large repositories, the set of visited nodes can consume significant memory. The walker is designed to keep node representations small (minimal identity information only) to reduce memory overhead. + +### WalkVisitor Trait + +The `WalkVisitor` trait defines callbacks invoked during graph traversal: + +**`start_node()`:** Called before loading a node's data. This allows setup operations like configuring sampling for blobstore accesses. + +**`visit()`:** Called after a node is loaded and expanded. This receives the node, its data, and its outgoing edges. Visitors can: +- Filter edges to prevent revisits +- Validate node data and edge consistency +- Perform operations on node data (scrubbing, corpus extraction, validation) +- Log or record information about nodes + +Different subcommands implement different visitors. For example, `ScrubVisitor` validates storage durability, `ValidatingVisitor` checks data consistency, and `CorpusVisitor` extracts blobs to disk. + +### Sampling Support + +For large repositories, processing the entire graph may be impractical due to memory constraints or time limits. The walker supports sampling operations on subsets of data. + +**Sampling Strategies:** + +- **Node hash sampling:** Select nodes based on their hash values. This provides stable sampling where the same nodes are selected across runs. + +- **Repository path sampling:** Select nodes based on repository paths using hash or regex patterns. Path sampling may be non-deterministic when multiple paths reference the same content. + +**Sampling Implementation:** + +The `SamplingWalkVisitor` combines high-level node sampling with low-level blobstore sampling. It sets a `SamplingKey` in the `CoreContext`, which the `SamplingBlobstore` uses to decide whether to sample individual blobstore operations. + +The `--sample-rate` and `--sample-offset` flags divide the repository into slices. By incrementing the offset, the entire repository can be processed one slice at a time. This allows large-scale operations to be distributed across multiple walker invocations or run incrementally. + +Visit tracking state can be maintained across slices to prevent re-visits when processing a repository in batches. + +## Memory Management + +Memory usage is a primary design constraint for the walker. Two factors drive memory consumption: + +**Visit Tracking:** To prevent cycles, the walker must track all visited nodes. For repositories with hundreds of millions of nodes, this requires efficient concurrent hash maps and minimal node representations. + +**Pending Queue:** As the graph expands, the queue of nodes waiting to be processed can grow to millions of entries. The size of the `Node` enum representation directly affects queue memory usage. Paths associated with nodes (for corpus extraction and routing) also contribute to memory overhead. + +**Optimization Strategies:** + +The walker employs several strategies to manage memory: + +- Keep `Node` representations minimal (identity information only, without full data) +- Separate `Node` (identity) from `NodeData` (loaded content) +- Use concurrent hash maps for visit tracking (trade-offs between memory and concurrency) +- Support sampling to process subsets of large repositories +- Stream large manifest contents rather than loading them entirely into memory + +Further optimizations under consideration include interning paths and nodes to reduce duplication, returning iterators instead of collected vectors, and extending bounded traversal to adjust queue sizes based on node type characteristics. + +## Logging and Monitoring + +The walker provides operational visibility through metrics and logging. + +**ODS Metrics:** The walker publishes counters and timeseries metrics for: +- Nodes visited per node type +- Edges traversed per edge type +- Validation errors detected +- Scrub operations performed +- Corpus blobs extracted + +These metrics power dashboards and alerting for continuous validation operations. + +**Scuba Logging:** The walker logs detailed information to Scuba tables: +- Validation errors with source node context +- Scrub issues with blobstore details +- Route information showing paths through the graph +- Sampling coverage and statistics + +Scuba logs support debugging and historical analysis of validation runs. + +**Progress Logging:** When running locally or in integration tests, the walker provides progress output via glog, showing traversal statistics and real-time status. + +## Operational Usage + +The walker serves multiple operational purposes: + +**Continuous Validation:** Run as a background job to continuously validate repository integrity. This detects data corruption, missing data, or derivation bugs. + +**Storage Verification:** Use scrub mode to verify multiplexed blobstore consistency. Schedule regular scrub runs to detect and repair storage inconsistencies before they impact availability. + +**Data Analysis:** Extract corpus subsets for offline analysis. Experiment with compression techniques, analyze repository structure, or test storage optimizations using real repository data. + +**Incident Response:** Run on-demand validation or scrubbing during incidents to assess impact and verify repairs. The walker can validate specific changesets or time ranges rather than entire repositories. + +**Capacity Planning:** Use compression-benefit analysis to understand storage characteristics and plan capacity. Measure the effectiveness of compression strategies before deploying them. + +## Integration with Storage Systems + +The walker integrates with Mononoke's storage architecture at multiple levels: + +**Blobstore Integration:** The walker accesses blobs through the standard blobstore interface. This means it benefits from cacheblob caching, respects redaction policies via redactedblobstore, and can sample operations via samplingblob. + +**Scrub Integration:** The scrub subcommand uses `ScrubBlobstore`, which provides access to individual component blobstores in a multiplex configuration. This allows verification and repair at the component level. + +**Healer Coordination:** While the blobstore healer job continuously processes the write-ahead log, the walker's scrub mode can detect issues that don't appear in the WAL (such as corruption or data loss in a single backend). The two systems are complementary. + +**Metadata Database:** The walker reads VCS mappings, bookmark data, and other metadata from SQL tables. This allows it to validate consistency between blobstore data and database state. + +## Configuration and Execution + +The walker runs using the standard `cmdlib/mononoke_app/` framework. It accepts configuration for: + +**Repository Selection:** Specify which repositories to process. Sharded execution can distribute repositories across multiple walker instances. + +**Traversal Parameters:** +- Starting points (specific bookmarks, changesets, or tags) +- Node and edge type filters to limit traversal scope +- Sampling rate and offset for partial traversal +- Enable/disable derivation (production runs disable to prevent writes) + +**Subcommand Options:** +- Output directory for corpus extraction +- Repair mode for scrub operations +- Validation strictness levels +- Compression parameters for compression-benefit analysis + +**Execution Modes:** +- Continuous tailing of bookmarks for new commits +- One-time traversal of specified ranges +- Dry-run mode for validation without side effects + +## Related Components + +The walker interacts with several other Mononoke components: + +**Blobstore Healer** (`jobs/blobstore_healer/`) - Continuously heals storage inconsistencies from the multiplexed blobstore WAL. The walker's scrub complements this by detecting issues outside the WAL. + +**Admin Tool** (`tools/admin/`) - Provides commands for on-demand validation and data inspection. Some admin subcommands use walker-like traversal internally. + +**Bounded Traversal** (`common/bounded_traversal/`) - The core traversal primitive used by the walker. This library provides efficient parallel tree/graph traversal with bounded concurrency. + +**Derived Data System** (`derived_data/`) - The walker validates derived data consistency. It can verify that derived data is correctly computed from Bonsai changesets. + +## Documentation + +Detailed implementation documentation is available in the walker source: + +- `jobs/walker/src/README.md` - Comprehensive design documentation covering graph representation, memory management, sampling, and implementation details +- `jobs/walker/src/` - Source code with inline documentation +- Integration tests in `tests/integration/` demonstrate walker usage patterns + +## Summary + +The walker is a graph traversal tool that validates Mononoke's data integrity, verifies storage durability, and supports data analysis operations. It models Mononoke's data structures as a dynamically-discovered graph and uses bounded traversal to process the graph efficiently. + +The four subcommands—scrub, validate, corpus, and compression-benefit—serve distinct operational purposes from storage verification to data analysis. Sampling support allows the walker to handle large-scale repositories by processing subsets of data, making it applicable to repositories of any size. + +The walker operates both as a continuous validation job and as an on-demand tool for incident response and analysis. Its integration with Mononoke's storage architecture and monitoring systems makes it a component of operational data integrity verification. + +For storage architecture details, see [Storage Architecture](2.4-storage-architecture.md). For background job context, see [Jobs and Background Workers](3.2-jobs-and-background-workers.md). diff --git a/eden/mononoke/docs/6.3-monitoring-and-observability.md b/eden/mononoke/docs/6.3-monitoring-and-observability.md new file mode 100644 index 0000000000000..149b1ff4a0cc2 --- /dev/null +++ b/eden/mononoke/docs/6.3-monitoring-and-observability.md @@ -0,0 +1,424 @@ +# Monitoring and Observability + +This document describes the monitoring and observability infrastructure in Mononoke. These systems provide visibility into server health, performance, and operational behavior. + +## Overview + +Mononoke exports metrics, logs, and traces to support operational monitoring and debugging. The observability infrastructure is designed to handle high-volume production traffic while providing detailed diagnostics when needed. + +The monitoring system is organized into several layers: + +- **Metrics** - Quantitative measurements exported to ODS (Operational Data Store) +- **Logging** - Structured logs written to Scuba for analysis +- **Tracing** - Request-level tracking using the tracing framework +- **Health checks** - Service health reporting via HTTP and FB303 +- **Performance counters** - Per-request operation metrics + +## Metrics (ODS Integration) + +Mononoke exports operational metrics using the `stats` crate. Metrics are exported to ODS for visualization and alerting. + +### Metric Types + +**Counters and Timeseries** +```rust +define_stats! { + prefix = "mononoke.edenapi.request"; + total_requests: timeseries(Rate, Sum), + requests: dynamic_timeseries("{}.requests", (method: String); Rate, Sum), +} +``` + +These track request rates, success counts, and error counts. Dynamic timeseries allow metrics to be broken down by dimensions like method name or repository. + +**Histograms** +```rust +files2_duration_ms: histogram(100, 0, 5000, Average, Sum, Count; P 50; P 75; P 95; P 99), +``` + +Histograms measure distributions of values such as request latency, response size, or operation duration. They report percentiles (P50, P95, P99) for understanding tail latency. + +### Metric Locations + +Metrics are defined in components throughout Mononoke: + +**Server Metrics** (`edenapi_service/src/middleware/ods.rs`, `git_server/`, `lfs_server/`) +- Request duration by endpoint +- Success and failure rates (4xx, 5xx responses) +- Response bytes sent +- Request load + +**Background Job Metrics** (`jobs/walker/`, `jobs/blobstore_healer/`) +- Items processed (commits walked, blobs healed) +- Validation pass/fail counts +- Scrub repair operations +- Queue depths and processing latency + +**Storage Metrics** (blobstore implementations) +- Blob gets, puts, and presence checks +- Cache hit/miss rates +- Multiplex operations +- Packblob compression ratios + +**Feature Metrics** (`pushrebase/`, `features/`) +- Pushrebase duration and conflicts +- Cross-repo sync operations +- Derived data derivation latency + +### Using Metrics + +Metrics are updated by calling methods on the generated `STATS` object: + +```rust +STATS::total_requests.add_value(1); +STATS::files2_duration_ms.add_value(duration_ms); +STATS::failure_5xx.add_value(1, (method_name.to_string(),)); +``` + +The stats framework aggregates these values and exports them to ODS. Dashboards and alerts are built on these metrics. + +## Logging (Scuba) + +Scuba is Mononoke's primary structured logging system. Scuba samples (log entries) contain key-value pairs describing operations, requests, and events. + +### CoreContext and Scuba + +Every Mononoke operation carries a `CoreContext` that includes a Scuba sample builder. The context flows through the request lifecycle, accumulating fields: + +```rust +pub struct CoreContext { + fb: FacebookInit, + session: SessionContainer, + logging: LoggingContainer, +} +``` + +The logging container holds: +- `MononokeScubaSampleBuilder` - Accumulates fields for the final Scuba sample +- Performance counters - Operation metrics +- Scribe client - For writing to Scribe streams + +### Scuba Fields + +Scuba samples typically include: + +**Request Metadata** +- Session ID +- Client info (hostname, username, identities) +- Repository name +- Request method and parameters + +**Performance Data** +- Request duration +- Blob operations (gets, puts) +- Cache statistics +- Bytes transferred + +**Outcome Information** +- Success or failure status +- Error messages +- Result sizes + +### Verbosity Levels + +The observability framework supports configurable verbosity (`observability/`): + +**Normal Level** - Logs all normal operations. This is the default level for production traffic. + +**Verbose Level** - Logs additional detail for debugging. Verbose logging can be enabled globally or selectively based on: +- Session ID +- Unix username +- Source hostname (via regex) + +The `ObservabilityContext` (in `observability/src/context.rs`) determines whether a sample should be logged based on its verbosity level and the current configuration. This allows detailed logging for specific users or sessions without overwhelming the logging system. + +### Scuba Tables + +Different components write to different Scuba tables: + +- **mononoke_edenapi** - EdenAPI protocol requests +- **mononoke_scs** - SCS Thrift API calls +- **mononoke_git** - Git protocol operations +- **mononoke_lfs** - LFS operations +- **mononoke_walker** - Walker validation and scrubbing +- **mononoke_backsyncer** - Cross-repo sync events + +Each table has component-specific fields reflecting the operation types and parameters. + +## Tracing + +Mononoke uses the `tracing` crate for structured logging within components. Tracing provides hierarchical logging with spans and events. + +### Usage + +Tracing calls are embedded throughout the code: + +```rust +use tracing::{info, warn, debug, error}; + +info!("Starting derivation for changeset {}", cs_id); +warn!("Slow operation detected: {}ms", duration_ms); +debug!(keys = ?blob_keys, "Fetching blobs"); +``` + +Spans group related operations: + +```rust +use tracing::Instrument; + +async fn derive_data(ctx: &CoreContext, cs_id: ChangesetId) -> Result<()> { + async move { + // derivation work + } + .instrument(tracing::info_span!("derive_data", ?cs_id)) + .await +} +``` + +Tracing integrates with the request context and can include correlation IDs for following a request through multiple services. + +### Log Levels + +**Error** - Unexpected failures requiring attention + +**Warn** - Recoverable issues or concerning patterns + +**Info** - Normal operational events + +**Debug** - Detailed diagnostic information + +Log output is configured per deployment and can be directed to local logs or centralized logging systems. + +## Request Context and Tracking + +The `CoreContext` (`server/context/`) serves as the request tracking mechanism. + +### Context Contents + +**Session Information** (`SessionContainer`) +- Session class (user, background, backup) +- Client metadata +- Permission checker +- Identity set + +**Logging Container** +- Scuba sample builder +- Performance counters stack +- Scribe client +- Sampling key + +**Metadata** +- Client request info (entry point, correlator) +- Source hostname +- Unix username + +### Context Flow + +A context is created when a request enters Mononoke (typically in the protocol server). The context is cloned and passed through: + +1. Protocol handlers (EdenAPI, Git, SCS) +2. API layer (`mononoke_api/`) +3. Features (pushrebase, hooks, etc.) +4. Repository facets +5. Storage operations + +Each layer can add fields to the Scuba sample or update performance counters. When the request completes, the accumulated data is logged. + +### Context Operations + +**Clone and Reset** +```rust +let new_ctx = ctx.clone_and_reset(); +``` +Creates a new context with reset performance counters, useful for sub-operations. + +**Fork Performance Counters** +```rust +let counters = ctx.fork_perf_counters(); +``` +Creates a snapshot of current performance counters for parallel operations. + +**Mutate Scuba Sample** +```rust +let new_ctx = ctx.with_mutated_scuba(|scuba| { + scuba.add("field_name", value) +}); +``` +Adds fields to the Scuba sample for the context. + +## Performance Counters + +Performance counters track detailed operation metrics within a request. Counters are defined in `server/context/src/perf_counters.rs`. + +### Counter Types + +**Blobstore Operations** +- `BlobGets`, `BlobPuts`, `BlobPresenceChecks` +- `BlobGetsMaxLatency`, `BlobPutsMaxLatency` +- `BlobGetsTotalSize`, `BlobPutsTotalSize` +- `BlobGetsDeduplicated`, `BlobPutsDeduplicated` + +**Caching** +- `CachelibHits`, `CachelibMisses` + +**Protocol-Specific** +- `EdenapiFiles`, `EdenapiTrees` +- `GetpackNumFiles`, `GettreepackNumTreepacks` +- `GetbundleNumCommits`, `GetbundleNumManifests` + +**Data Transfer** +- `BytesSent` + +Performance counters are accumulated in the `PerfCountersStack` and can be nested for tracking operations within sub-operations. When a request completes, counters are exported to Scuba for analysis. + +### Counter Access + +Operations access counters through the CoreContext: + +```rust +let perf_counters = ctx.perf_counters(); +``` + +The blobstore layer automatically updates blob operation counters. Protocol handlers update protocol-specific counters based on the data served. + +## Health Checks + +Mononoke servers expose health check endpoints for monitoring and load balancing. + +### HTTP Health Checks + +The main server (`server/repo_listener/`) responds to health check requests: + +``` +GET / -> "I_AM_ALIVE" +GET /health_check -> "I_AM_ALIVE" +``` + +If the server is shutting down, these endpoints return "EXITING". Load balancers use these endpoints to determine which servers should receive traffic. + +### FB303 Service + +Mononoke applications integrate with FB303 (`cmdlib/mononoke_app/src/monitoring.rs`), a service framework that provides: + +**Status Reporting** +- `FbStatus::Alive` - Server is ready +- `FbStatus::Starting` - Server is initializing +- `FbStatus::Stopping` - Server is shutting down + +The `ReadyFlagService` implementation starts in the `Starting` state and transitions to `Alive` once initialization completes. + +**Thrift Interface** +FB303 exposes a Thrift interface on a configured port (via `--fb303-thrift-port`). This interface allows monitoring systems to: +- Query server status +- Retrieve counter values +- Check build information + +**Prometheus Export** +In fbcode builds, FB303 metrics can be exported to Prometheus format via the `--prometheus-host-port` flag. This enables integration with Prometheus-based monitoring stacks. + +### Monitoring Framework + +The `mononoke_app` framework (`cmdlib/mononoke_app/`) initializes monitoring automatically. Applications using this framework receive: +- FB303 service +- Stats aggregation +- Graceful shutdown handling +- Health check integration + +## Common Patterns + +### Request Logging + +Protocol servers use middleware to log requests: + +**Log Middleware** (`gotham_ext/src/middleware/log.rs`) +Logs HTTP requests and responses: +``` +IN > GET /repo/trees +OUT < 200 150ms 1024bytes +``` + +**Scuba Middleware** (`gotham_ext/src/middleware/scuba.rs`) +Constructs and logs Scuba samples for each request with timing, status, and metadata. + +**ODS Middleware** (`edenapi_service/src/middleware/ods.rs`) +Updates ODS metrics for request duration, success/failure, and response size. + +### Operational Dashboards + +Metrics exported to ODS are visualized in operational dashboards. Common dashboard categories: + +**Service Health** +- Request rate and error rate +- P50, P95, P99 latency +- Success vs. failure breakdown + +**Resource Usage** +- Blobstore operation rates +- Cache hit rates +- Bytes transferred + +**Feature-Specific** +- Pushrebase operations and conflicts +- Derivation latency and backlog +- Cross-repo sync lag + +**Background Jobs** +- Walker progress and error rates +- Healer repair operations +- Statistics collection status + +### Query Patterns + +**Scuba Queries** +Scuba samples can be queried to analyze specific requests, debug failures, or identify performance patterns: +- Filter by session ID to trace a specific client session +- Filter by repository to analyze repository-specific behavior +- Aggregate by endpoint to identify slow operations +- Join with performance counters to correlate latency with blob operations + +**ODS Queries** +ODS timeseries support aggregation and alerting: +- Monitor P99 latency for SLA compliance +- Track error rates for alerting +- Compare metrics across deployments +- Analyze capacity and scaling needs + +## Integration with Deployment + +Monitoring configuration is specified via command-line arguments and configuration files: + +**Command-Line Flags** +- `--fb303-thrift-port` - Enable FB303 service +- `--prometheus-host-port` - Export Prometheus metrics +- `--scuba-dataset` - Scuba table name +- `--cache-mode` - Caching configuration (affects cache metrics) + +**Configuration Files** +Observability configuration (`scm/mononoke/observability/observability_config`) controls: +- Scuba verbosity levels +- Verbose sessions/usernames +- Sampling rates + +Configuration is loaded via `cached_config` and can be updated without restarting servers. + +## Relationship to Architecture + +The monitoring system reflects Mononoke's layered architecture: + +**Service Layer** - HTTP middleware logs requests, updates per-endpoint metrics + +**API Layer** - Scuba samples include high-level operation types + +**Feature Layer** - Features log specific events (pushrebase conflicts, derivation completion) + +**Repository Layer** - Facets update performance counters (blob operations, cache hits) + +**Storage Layer** - Blobstore implementations track latency and throughput + +This layering allows monitoring at multiple granularities, from high-level service health to detailed storage operations. + +## Related Documentation + +- [Architecture Overview](1.3-architecture-overview.md) - How monitoring fits into the overall system +- [Jobs and Background Workers](3.2-jobs-and-background-workers.md) - Job-specific monitoring +- [Servers and Services](3.1-servers-and-services.md) - Server health checks and metrics diff --git a/eden/mononoke/docs/A.1-better-engineering.md b/eden/mononoke/docs/A.1-better-engineering.md new file mode 100644 index 0000000000000..637d7ed021786 --- /dev/null +++ b/eden/mononoke/docs/A.1-better-engineering.md @@ -0,0 +1,453 @@ +# Better Engineering + +This document tracks long-term engineering improvements and technical debt that could be addressed in the Mononoke codebase. These are substantive improvements identified through codebase analysis, not immediate action items. + +**Note:** These were mostly generated by automated analysis and may not be relevant or applicable to the current codebase. Please use your judgment when considering these items. + +--- + +## Bonsai Data Model and Core Types + +### Remove `BonsaiChangeset::git_tree_hash` + +**Location:** `mononoke_types/src/bonsai_changeset.rs:57` + +**Problem:** This was added as the first attempt of storing refs to trees. We now use `git_ref_content_mapping`. The validation logic enforces constraints that make this field mutually exclusive with normal changeset data and with `git_annotated_tag`. + +**Improvement:** Migrate git_tree_hash to a separate mapping table (similar to `bonsai_git_mapping`) or create a distinct type for git tree representations rather than overloading BonsaiChangeset. + +**Impact:** Reduces complexity in core type validation, eliminates special-case handling, improves type safety by separating concerns. + +### Consolidate Duplicated VCS Mapping Trait Pattern + +**Location:** `repo_attributes/bonsai_*_mapping/src/lib.rs` (8 different directories) + +**Problem:** The codebase has 6+ nearly identical mapping traits (BonsaiHgMapping, BonsaiGitMapping, BonsaiGlobalrevMapping, BonsaiSvnrevMapping, BonsaiTagMapping) with duplicated conversion methods and nearly identical `BonsaiOrX` enum types. + +**Improvement:** Create a generic `BonsaiMapping` trait or macro to eliminate duplication. The common pattern could be abstracted into a single parameterized implementation with type-specific hooks only where needed. + +**Impact:** Reduces maintenance burden significantly (changes need to be made in 6+ places currently), improves consistency across VCS backends, easier to add new mapping types. + +### Restructure FileChange Enum + +**Location:** `mononoke_types/src/file_change.rs:101-107` + +**Problem:** The `FileChange` enum has four variants (Change, Deletion, UntrackedChange, UntrackedDeletion) where the tracked/untracked distinction creates duplication. The `simplify()` method exists specifically to collapse this distinction, suggesting the separation may be too granular. There's a TODO about adding copy information to untracked changes. + +**Improvement:** Restructure as a struct with `change_type`, `tracking`, and optional `data` fields. This would make adding copy_from to untracked changes trivial and eliminate the need for `simplify()`. + +**Impact:** More flexible type that's easier to extend, eliminates redundant enum variants, resolves TODO about copy info. + +### Eliminate Clone Overhead in Unode ID Computation + +**Location:** `mononoke_types/src/unode.rs:116,235` + +**Problem:** Both `FileUnode::get_unode_id()` and `ManifestUnode::get_unode_id()` have "FIXME: try avoid clone" comments. The methods clone the entire structure just to compute its ID. + +**Improvement:** Cache the ID when the unode is created/deserialized, or implement `BorrowedBlobstoreValue` trait that can compute IDs from references. + +**Impact:** Performance improvement (eliminates unnecessary allocations/copies), cleaner API, resolves long-standing FIXMEs. + +### Consolidate Path Type Proliferation + +**Location:** `mononoke_types/src/path.rs` and `path/mpath_element.rs` + +**Problem:** Overlapping path types with subtle differences: `MPath`, `NonRootMPath`, `RepoPath`, `MPathElement`, `MPathHash`. The relationship creates conversion overhead and cognitive load. + +**Improvement:** Consolidate to 2-3 core types with clearer roles using const generics or the type system for root/non-root distinction. + +**Impact:** Reduced cognitive load, fewer conversions, clearer type relationships. + +--- + +## Repository Facets and Composition + +### Reduce Trait Bound Verbosity + +**Location:** Throughout `features/`, `repo_attributes/bookmarks/bookmarks_movement/` + +**Problem:** Features define trait aliases requiring 10-20+ individual facet refs, creating verbose and error-prone code. + +**Improvement:** Create higher-level facet groupings like `RepoStorageFacets`, `RepoVcsFacets`, `RepoPermissionFacets`, `RepoDerivedDataFacets`. + +**Impact:** Reduces trait bound lists from 15+ items to 3-5 semantic groups, easier to understand, faster compilation, clearer grouping for new facets. + +### Standardize Arc vs Ref Trait Suffixes + +**Location:** All facets in `repo_attributes/` + +**Problem:** Inconsistent patterns - some have only `XxxRef`, some only `XxxArc`, some have both. Features randomly mix these. + +**Improvement:** Standardize on a single pattern (preferably only `Ref` variants for cleaner API) with `Arc` used only where required. + +**Impact:** Eliminates confusion, reduces cognitive load, makes refactoring safer, smaller binary size. + +### Split RepoDerivedData Facet + +**Location:** `repo_attributes/repo_derived_data/src/lib.rs` + +**Problem:** `RepoDerivedData` violates single responsibility by containing configuration, multiple managers, selection logic, 7+ boilerplate methods, bubble context management, and scuba mutation. + +**Improvement:** Split into multiple facets: `RepoDerivedDataConfig`, `RepoDerivedDataManager`, `RepoDerivedDataRegistry`. Use builder pattern for overrides. + +**Impact:** Easier to mock in tests, clearer separation of concerns, reduces code duplication (150+ lines of boilerplate). + +--- + +## Derived Data Framework + +### Eliminate Duplicate get_file_changes Implementations + +**Location:** `derived_data/{fsnodes,unodes,skeleton_manifest}/mapping.rs` + +**Problem:** The same `get_file_changes()` function is duplicated in at least 3 derived data types with identical implementations. + +**Improvement:** Move to a shared utility module (e.g., `derived_data/src/common.rs`) and reuse. + +**Impact:** Reduces ~50 lines of duplication, ensures consistent behavior, single place to optimize. + +### Abstract Mapping Storage Boilerplate + +**Location:** All `mapping.rs` files (17+ files) + +**Problem:** Every derived data type manually implements `format_key()`, `store_mapping()`, and `fetch()` with nearly identical patterns. + +**Improvement:** Create a `derive_mapping!` macro or trait-based abstraction that generates these methods given just the key prefix. + +**Impact:** Eliminates ~100+ lines of boilerplate per type, prevents copy-paste errors, easier versioning/key format changes. + +### Centralize Hardcoded Buffer/Concurrency Constants + +**Location:** Throughout framework + +**Problem:** Magic numbers scattered across codebase (`buffer_unordered(64)`, `buffered(100)`, `DEFAULT_BATCH_SIZE: u64 = 20`) with no configuration or tuning capability. + +**Improvement:** Add a `DerivationTuning` config struct in `DerivationContext` with justknobs-backed defaults for buffer sizes, batch sizes, concurrency limits. + +**Impact:** Enables production tuning without code changes, better performance optimization, clearer intent. + +### Deprecate and Remove Skeleton Manifest V1 + +**Location:** `derived_data/skeleton_manifest/` + +**Problem:** Multiple skeleton manifest versions (v1, v2) with substantially duplicated derivation logic. + +**Improvement:** Create deprecation plan for old versions. + +**Impact:** Removes duplicated code. + +### Standardize Batch Derivation Strategies + +**Location:** Various `derive_batch` implementations + +**Problem:** No clear guidance on when to use sequential vs parallel vs stacked batch derivation. Each type implements differently. + +**Improvement:** Create documented batch derivation patterns with decision tree. Provide template implementations. + +**Impact:** Better batch performance for types not currently optimized, consistency, easier to implement new types correctly. + +### Expand derive_from_predecessor Pattern + +**Location:** Only in 5 derived data types + +**Problem:** The "predecessor optimization" is extremely valuable for backfilling but only 5 types use it. No documentation or guide for implementing. + +**Improvement:** Document the pattern in framework docs. Create guide/checklist for when it's applicable. Add helper methods to make implementation easier. Identify candidates among existing types. + +**Impact:** 10-100x faster backfilling for large repos when enabled, clearer migration paths between versions. + +### Implement Outstanding Performance TODOs + +**Location:** `deleted_manifest/derive_batch.rs:224`, `inferred_copy_from/derive.rs:172`, `mercurial_derivation/src/derive_hg_changeset.rs:264` + +**Problem:** Known optimization opportunities documented as TODOs but not prioritized. Actual performance impact unknown. + +**Improvement:** Benchmark each TODO to quantify impact. Create tasks for high-impact ones. Either implement or remove low-value TODOs. + +**Impact:** Measured performance improvements (potentially 20-50% for impacted paths), cleaner codebase. + +### Create Shared Thrift Serialization Framework + +**Location:** `from_thrift()` and `into_thrift()` implementations in all 17+ mapping files + +**Problem:** Repeated pattern of match statements with nearly identical error messages. Error handling is verbose and duplicated. + +**Improvement:** Create `derive_thrift_mapping!` macro that generates these methods with less boilerplate. Consider auto-generation from thrift definitions. + +**Impact:** Reduces ~30 lines per type, consistent error messages, easier to add new serialization formats. + +### Unify Testing Infrastructure + +**Location:** Test modules across all derived data types + +**Problem:** Each type's tests define nearly identical `TestRepo` facet containers. Test utilities follow similar patterns but are duplicated. + +**Improvement:** Create `derived_data_test_utils` with common test repo factory and trait-based test helpers. + +**Impact:** Reduces test code by ~40%, ensures consistent test coverage, easier to add new types with good tests. + +--- + +## Storage Architecture + +### Simplify Decorator Stack Configuration + +**Location:** `blobstore/factory/src/blobstore.rs:451-628` + +**Problem:** The `make_blobstore` function recursively applies decorators with hardcoded order and `needs_wrappers` flag. Hard to understand which decorators will be applied, difficult to test interactions. + +**Improvement:** Create a `DecoratorBuilder` pattern that explicitly documents the decorator stack and makes order testable and customizable. + +**Impact:** Makes decorator ordering visible and documented, easier to add/remove/reorder, testable decorator combinations. + +### Reduce BlobstoreOptions Proliferation + +**Location:** `blobstore/factory/src/blobstore.rs:70-181` + +**Problem:** `BlobstoreOptions` contains 8+ different option structs bundled together even when only 1-2 are needed. Creates excessive cloning and tight coupling. + +**Improvement:** Use builder pattern with trait-based options or split into logical groups (StorageOptions, ObservabilityOptions, TestingOptions). + +**Impact:** Reduce unnecessary cloning, only pass relevant options, easier to add new options. + +### Clarify Scrub Configuration + +**Location:** `blobstore/multiplexedblob/src/scrub.rs:93-110` + +**Problem:** `ScrubOptions` has grown to include many modes with subtle interactions. `ScrubWriteOnly` enum has 4 variants with complex semantics. + +**Improvement:** Split into separate concerns: `ScrubPolicy`, `WriteOnlyStrategy`, `TimingConfig`. Use type-state pattern to prevent invalid configurations. + +**Impact:** Prevent misconfiguration, easier to understand scrubbing behavior, independently testable concerns. + +### Clarify PutBehaviour vs OverwriteStatus + +**Location:** `blobstore/src/lib.rs:454-513` + +**Problem:** Two related but distinct types (`PutBehaviour` and `OverwriteStatus`) where the relationship isn't obvious. Some decorators don't preserve status correctly. + +**Improvement:** Rename to make relationship clearer (`PutPolicy` and `PutOutcome`). Add documentation showing the state machine. Add debug assertions in decorators. + +**Impact:** Ensure put guarantees are maintained through decorators, easier to trace operations, self-documenting code. + +### Improve Blobstore Enumeration + +**Location:** `blobstore/src/lib.rs:515-609` + +**Problem:** `BlobstoreKeySource` enumeration has decorators manually stripping/adding key transformations. No support for pagination limits or filtering. Continuation tokens are opaque. + +**Improvement:** Add a `KeyTransform` trait that decorators implement. Support server-side filtering. Make continuation tokens structured. Add `limit` parameter. + +**Impact:** Reduce data transfer, prevent OOM from enumerating millions of keys, debuggable continuation tokens. + +--- + +## Server and API Layer + +### Unify RequestContextMiddleware Implementations + +**Location:** `lfs_server/src/middleware/request_context.rs`, `gotham_ext/src/middleware/request_context.rs` + +**Problem:** LFS server has its own implementation with custom method tracking, while git_server uses the generic version. Code duplication (~140+ lines), inconsistent authentication enforcement. + +**Improvement:** Create unified, configurable `RequestContextMiddleware` in gotham_ext that supports optional method tracking, consistent authentication, and unified session building. + +**Impact:** Reduces ~200 lines of duplication, ensures consistent auth behavior, easier to add new servers. + +### Standardize Error Formatting Across Servers + +**Location:** `git_server/src/service/error_formatter.rs`, `lfs_server/src/service/error_formatter.rs`, edenapi_service + +**Problem:** Each server implements its own error formatter - git returns plain text, LFS returns JSON with request_id, inconsistent error detail exposure. + +**Improvement:** Create shared error formatting library with protocol-aware formatters (Git, LFS, JSON), consistent request_id inclusion, and configurable detail exposure. + +**Impact:** Consistent error responses across all services, better debugging with request_id everywhere, easier to add structured logging. + +### Create Shared Health Check Handler + +**Location:** `git_server/src/service/router.rs:101`, `lfs_server/src/service/router.rs:80` + +**Problem:** Different health check implementations - git has special wait-time header, LFS returns "EXITING" when shutting down, inconsistent for load balancers. + +**Improvement:** Create shared health check handler in gotham_ext with support for optional will_exit state, delay header, consistent response format, and ready vs alive probes. + +**Impact:** Consistent load balancer behavior, reusable logic (~30 lines per server), better graceful shutdown. + +### Eliminate Repo Sharding Logic Duplication + +**Location:** `server/src/main.rs:101-215`, `git_server/src/sharding.rs:23-105`, `scs/scs_server/src/main.rs:144-221` + +**Problem:** Each server implements nearly identical sharding logic (~100 lines duplicated 3+ times). + +**Improvement:** Create a generic `GenericRepoShardedProcess` in cmdlib/sharding. + +**Impact:** Eliminates ~300 lines of duplication, single place to fix sharding bugs, easier to add new sharded services. + +### Unify TLS Acceptor Setup + +**Location:** `server/src/main.rs:253-275`, `git_server/src/main.rs:195-207`, `lfs_server/src/main.rs:213-225` + +**Problem:** Nearly identical TLS acceptor building code (~20-30 lines duplicated per server). + +**Improvement:** Add helper method to `TLSArgs` like `build_acceptor(&self, alpn_protocols: Option<&[u8]>) -> Result`. + +**Impact:** Reduces ~60-80 lines of duplication, ensures consistent TLS configuration, single point for security updates. + +### Clarify Server Initialization Patterns + +**Location:** HTTP vs Thrift server initialization + +**Problem:** Two different termination methods (`run_until_terminated` vs `wait_until_terminated`) with different signatures and shutdown coordination patterns. + +**Improvement:** Unify into a single termination pattern or clearly rename methods (`run_http_server_until_terminated` vs `wait_for_termination`). Or create unified `ServerShutdown` builder. + +**Impact:** Clearer API for server authors, consistent shutdown behavior, reduces cognitive load. + +### Standardize Middleware Stack Ordering + +**Location:** `git_server/src/main.rs:300-332`, `lfs_server/src/main.rs:319-343`, `edenapi_service/src/lib.rs:80-110` + +**Problem:** Each server builds middleware stacks in different orders - TLS, Scuba, Load positions vary. Hard to understand why ordering differs, potential for bugs. + +**Improvement:** Create a `ServerMiddlewareBuilder` with documented middleware ordering and semantic groupings (core, auth, app, observability). + +**Impact:** Consistent middleware ordering across servers, prevents ordering bugs, ~40 lines of documentation vs scattered code. + +### Create Shared Repo Resolution Logic + +**Location:** Scattered across servers + +**Problem:** Repo name extraction scattered across servers with different path formats, no validation, inconsistent errors. + +**Improvement:** Create shared repo resolution utilities with `RepoResolver` trait for different path schemes, common validation, consistent errors, and support for aliases. + +**Impact:** Consistent repo not found errors, easier to add features like repo aliases, single place for validation, better security. + +### Add ConnectionSecurityChecker Helper + +**Location:** `git_server/src/main.rs:360-361`, `lfs_server/src/main.rs:356-357` + +**Problem:** Each HTTP server creates `ConnectionSecurityChecker` independently with same pattern duplicated 3+ times. + +**Improvement:** Add helper to MononokeApp: `pub async fn connection_security_checker(&self) -> Result`. + +**Impact:** Ensures consistent security checking, single place to update security logic, harder to forget checks in new servers. + +### Create Bound Address File Writing Utility + +**Location:** `git_server/src/main.rs:336-340`, `lfs_server/src/main.rs:348-352`, `scs/scs_server/src/main.rs:379-383` + +**Problem:** Identical bound address file writing code duplicated across all servers (~5 lines per server). + +**Improvement:** Create utility function: `pub fn write_bound_address(path: Option<&Path>, addr: &str) -> Result<()>`. + +**Impact:** Consistent file writing behavior, single place for error handling, could add features like atomic writes. + +--- + +## Testing Infrastructure + +### Split Monolithic Test Library File + +**Location:** `tests/integration/library.sh` (1,843 lines) + +**Problem:** Single 48KB file containing all test utilities. Difficult to navigate, maintain, and understand. No clear organization. + +**Improvement:** Split into focused modules (server-helpers.sh, repo-setup.sh, wait-utils.sh, admin-helpers.sh). Group related functions together. + +**Impact:** Better maintainability, easier onboarding, reduced merge conflicts, clearer ownership. + +### Consolidate wait_for_* Functions + +**Location:** `tests/integration/library.sh` (18+ wait functions) + +**Problem:** 18+ specialized wait functions with similar polling logic and slight variations. Hardcoded timeouts and sleep intervals. + +**Improvement:** Create generic `wait_for_condition` function with configurable condition check, timeout, poll interval with exponential backoff, and failure message. + +**Impact:** Faster tests (via exponential backoff), consistent timeout behavior, easier debugging, ~300 lines of code reduction. + +### Replace Hardcoded Sleep Calls + +**Location:** Throughout `library.sh` and test files + +**Problem:** 17+ hardcoded `sleep` calls with arbitrary durations. Fixed sleep durations cause unnecessary test slowness and race conditions. + +**Improvement:** Replace all fixed sleeps with condition-based waiting. Implement exponential backoff for all polling. Use inotify/file watching where possible. + +**Impact:** 20-30% faster test execution, more reliable tests, better failure diagnostics. + +### Standardize Test Repository Setup + +**Location:** Across 586 `.t` files + +**Problem:** Only 25 tests use `default_setup_*` helpers. Most manually initialize repos. Two competing methods (blobimport vs testtool_drawdag). Massive duplication of setup boilerplate. + +**Improvement:** Expand fixture library using Rust `TestRepoFixture` trait. Encourage fixture use via better documentation. Deprecate manual setup patterns. + +**Impact:** Reduced test setup code by 50%+, faster test authoring, consistent environments, easier improvements propagate to all tests. + +### Improve Test Failure Diagnostics + +**Location:** Error handling throughout integration tests + +**Problem:** When tests fail, often just see "timeout" or generic error. Server logs buried. No structured logging or failure context. Hard to diagnose which parallel operation failed. + +**Improvement:** Wrap all wait_for_* functions to automatically dump relevant logs on timeout. Add structured failure context. Implement test operation timing. Auto-capture server logs on failure. + +**Impact:** 50-80% reduction in debugging time, faster failure diagnosis, fewer "flaky test" retries. + +### Add Test Performance Tracking + +**Location:** 31 tests marked with `#require slow` + +**Problem:** 31 tests marked as "slow" but no performance tracking. No visibility into which operations are slow. Tests get slower over time without anyone noticing. + +**Improvement:** Implement test timing infrastructure that logs operation durations. Add performance regression detection (warn if test >20% slower than baseline). Profile slow tests and optimize. + +**Impact:** 2-5x speedup for slowest tests, prevent performance regression, better CI resource utilization. + +### Complete Shell/Python Migration + +**Location:** `library.sh` (32 uses of Python wrapper), `dbrtest_runner.py` + +**Problem:** Functions implemented in Python called via `python_fn` wrapper from shell. Fragile environment variable passing. TODOs indicate incomplete migration. Debugging is difficult. + +**Improvement:** Complete Python migration for functions flagged with TODO, or go fully shell-based. Eliminate `python_fn` wrapper complexity. Make language choice deliberate per-function. + +**Impact:** Simpler mental model, easier debugging, clearer maintenance ownership, eliminate TODOs. + +### Abstract Test-Specific Library Patterns + +**Location:** `library-push-redirector.sh` (22KB), `library-git-lfs.sh` (4KB) + +**Problem:** Large specialized library files that duplicate patterns from main library. Similar configuration generation logic duplicated. + +**Improvement:** Extract common patterns into shared utilities. Create declarative config builders instead of imperative JSON string construction. Use Rust test utilities for complex setup. + +**Impact:** Less duplication, easier to add new test scenarios, better config validation. + +### Add Missing Test Helpers + +**Location:** Repetitive patterns across `.t` files + +**Problem:** No helpers for common patterns like "push and verify bookmark moved", "make commit with specific files", manual jq parsing repeated hundreds of times, ad-hoc scuba log validation. + +**Improvement:** Add high-level test helpers like `push_and_verify_bookmark`, `create_commit_with_files`, `assert_scuba_log_contains`, `assert_bookmark_equals`. + +**Impact:** Tests become more readable, common errors prevented, faster test authoring. + +### Add Unit Tests for Test Infrastructure + +**Location:** `library.sh`, `dbrtest_runner.py`, test utilities + +**Problem:** 48KB of shell code with no unit tests. Test infrastructure bugs affect all 586 integration tests. Helper functions can break silently. + +**Impact:** Add unit tests for critical library functions. Create `test-library-*.t` tests. Test edge cases. Add CI job for library unit tests. + +**Impact:** Safer refactoring, prevent infrastructure regressions, faster development iteration. + +--- + +This document is a living collection of improvement opportunities. Items should be evaluated for impact and feasibility before implementation. Not all items will be implemented, and priorities may shift based on business needs. diff --git a/eden/mononoke/docs/README.md b/eden/mononoke/docs/README.md new file mode 100644 index 0000000000000..3bedd78ec5e8c --- /dev/null +++ b/eden/mononoke/docs/README.md @@ -0,0 +1,148 @@ +# Mononoke Documentation + +## About Mononoke + +Mononoke is a scalable source control server designed to handle massive repositories with millions of files, thousands of commits per day, and hundreds of thousands of branches. As the server-side component of Meta's Sapling source control ecosystem, Mononoke works together with EdenFS (a virtual filesystem) and the Sapling CLI to provide version control operations at large scale. + +Unlike traditional version control servers that are tied to a specific VCS, Mononoke uses a canonical, VCS-agnostic data model called Bonsai as its single source of truth. This enables Mononoke to serve both Sapling and Git clients from the same backend, converting between formats as needed while maintaining consistency. The architecture separates write operations from read operations, computes expensive indexes asynchronously, and uses horizontal scaling to handle load. + +For a complete introduction, see [What is Mononoke?](1.1-what-is-mononoke.md). + +## About This Documentation + +This documentation provides high-level architectural and conceptual information about Mononoke for developers, operators, and AI agents. It covers core architectural patterns, major components (servers, jobs, tools), key features, VCS integration, and operational aspects. + +**Not covered here:** Build/test workflows are in [CLAUDE.md](../CLAUDE.md). API documentation is in rustdoc comments. Detailed operational procedures are in component-specific directories. + +## Getting Started + +**New to Mononoke?** Start with Section 1 (Introduction). + +**Building or developing?** See [CLAUDE.md](../CLAUDE.md) for build instructions and testing procedures. + +**Looking for specific topics?** Use the table of contents below. + +## Documentation Contents + +### 1. Introduction + +*Core concepts and orientation for understanding Mononoke* + +**[1.1 - What is Mononoke?](1.1-what-is-mononoke.md)** +Mononoke's role as a scalable source control server, its place in the Sapling ecosystem, design goals, and deployment scenarios. + +**[1.2 - Key Concepts](1.2-key-concepts.md)** +Essential concepts: Bonsai (canonical data model), content addressing (hash-based storage), repository facets (trait-based composition), derived data (asynchronous indexes), blobstore (immutable storage), metadata database, and VCS mappings. + +**[1.3 - Architecture Overview](1.3-architecture-overview.md)** +System architecture (how services are composed) and code architecture (how applications are structured). Covers data flow, write vs. read paths, and how components fit together. + +**[1.4 - Navigating the Codebase](1.4-navigating-the-codebase.md)** +Guide to finding components in 70+ directories. Covers directory organization, BUCK files, and how to locate implementations and tests. + +### 2. Architecture + +*Deep dive into Mononoke's data model, patterns, and storage design* + +**[2.1 - Bonsai Data Model](2.1-bonsai-data-model.md)** +Mononoke's core data model: what Bonsai is, how it represents commits and file changes, content addressing, and how it enables multi-VCS support. + +**[2.2 - Repository Facets](2.2-repository-facets.md)** +The facet pattern used throughout Mononoke. Covers major facet categories (identity, storage, commit graph, derived data, VCS mappings, bookmarks, operations) and how features compose facets. + +**[2.3 - Derived Data](2.3-derived-data.md)** +The derived data framework: what derived data is, why it's computed off the write path, the derivation process, major types, and remote derivation. + +**[2.4 - Storage Architecture](2.4-storage-architecture.md)** +How Mononoke stores and caches data. Covers blobstore architecture (backends, decorator pattern), metadata database, caching strategy, and packblob compression. + +### 3. Components + +*Servers, background jobs, command-line tools, and shared libraries* + +**[3.1 - Servers and Services](3.1-servers-and-services.md)** +Main protocol servers (Mononoke/SLAPI, SCS, Git, LFS) and internal microservices (Land, Derived Data, Bookmark, Diff, Load Limiter). + +**[3.2 - Jobs and Background Workers](3.2-jobs-and-background-workers.md)** +Background maintenance tasks: Walker (graph validation), Blobstore Healer (storage durability), Derivation Worker, Cross-Repo Sync, and Statistics Collector. + +**[3.3 - Tools and Utilities](3.3-tools-and-utilities.md)** +Command-line tools: admin CLI (primary tool), import/export tools (blobimport, gitimport), verification tools (aliasverify), and maintenance utilities (packer, sqlblob_gc). + +**[3.4 - Libraries and Frameworks](3.4-libraries-and-frameworks.md)** +Shared libraries: cmdlib/mononoke_app framework for new binaries, common utilities (async, SQL, logging), core types, and testing utilities. + +### 4. Features + +*Key source control operations and workflows* + +**[4.1 - Pushrebase](4.1-pushrebase.md)** +Server-side rebasing for maintaining linear history at scale. Covers the pushrebase process, conflict detection, hooks integration, and use in Sapling and Git workflows. + +**[4.2 - Cross-Repo Sync](4.2-cross-repo-sync.md)** +Repository synchronization between large and small repos. Covers sync patterns, commit transformation, and sync job operation. + +**[4.3 - Hooks](4.3-hooks.md)** +Policy enforcement at push time. Covers hook types (bookmark, changeset, file), configuration, execution, and the hook manager facet. + +**[4.4 - Redaction](4.4-redaction.md)** +Content redaction for removing sensitive data. Covers how redaction works at the blobstore level and access control. + +**[4.5 - Microwave](4.5-microwave.md)** +Cache warming for improved performance. Covers what gets warmed (derived data, blobstore) and when warming happens. + +### 5. VCS Integration + +*Support for Git, Mercurial, and Sapling clients* + +**[5.1 - Git Support](5.1-git-support.md)** +Git protocol server, Bonsai ↔ Git conversion, Git-specific derived data, reference handling, LFS integration, and source of truth tracking. + +**[5.2 - Mercurial and Sapling Support](5.2-mercurial-sapling-support.md)** +Historical context, Bonsai ↔ Mercurial conversion, wire protocol, Mercurial-specific derived data, EdenAPI protocol, and compatibility requirements. + +### 6. Operations + +*Operational concerns: performance, validation, and observability* + +**[6.1 - Rate Limiting and Load Shedding](6.1-rate-limiting-and-load-shedding.md)** +Load management strategies, the Load Limiter service, QPS limits, and load shedding under pressure. + +**[6.2 - Walker and Validation](6.2-walker-and-validation.md)** +Graph traversal and validation tool. Covers scrubbing (validation and repair), corpus generation, and compression analysis. + +**[6.3 - Monitoring and Observability](6.3-monitoring-and-observability.md)** +Metrics (ODS/stats), logging (Scuba), tracing, dashboards, health checks, and performance indicators. + +### Appendix A. Future Improvements + +**[A.1 - Better Engineering](A.1-better-engineering.md)** +A living document tracking long-term engineering improvements and technical debt that could be addressed in the codebase. + +## Additional Resources + +**Development:** +- [CLAUDE.md](../CLAUDE.md) - Build commands, tests, build modes, development patterns + +**Code:** +- Mononoke codebase: `fbcode/eden/mononoke/` +- Open source: [Sapling on GitHub](https://github.com/facebook/sapling) + +**Related Projects:** +- EdenFS: `fbcode/eden/fs/` - Virtual filesystem +- Sapling CLI: `fbcode/eden/scm/` - Command-line interface +- Eden project docs: `fbcode/eden/CLAUDE.md` + +**External:** +- Sapling website: https://sapling-scm.com/ +- Oncall: `scm_server_infra` + +## Contributing + +Update these docs for architectural changes, new components, or component reorganization. Do not update for implementation details, bug fixes, or configuration changes—those belong in component directories, rustdoc, or CLAUDE.md. + +Maintain a professional, neutral, factual tone. Keep content high-level, cross-reference liberally, and verify accuracy against the code. + +--- + +For questions, contact the `scm_server_infra` oncall. diff --git a/eden/mononoke/edenapi_service/src/handlers/commit.rs b/eden/mononoke/edenapi_service/src/handlers/commit.rs index 2f18c634c5bfd..a569a1ba43cb3 100644 --- a/eden/mononoke/edenapi_service/src/handlers/commit.rs +++ b/eden/mononoke/edenapi_service/src/handlers/commit.rs @@ -218,10 +218,13 @@ async fn bump_counter_check_ratelimit( } }; + let atlas = ctx.metadata().clientinfo_atlas(); + let limit = match rate_limiter.find_rate_limit( Metric::CommitsPerUser, None, client_request_info.main_id.as_deref(), + atlas, ) { Some(limit) => limit, None => { diff --git a/eden/mononoke/edenapi_service/src/middleware/rate_limiter.rs b/eden/mononoke/edenapi_service/src/middleware/rate_limiter.rs index cf167df229482..9b72d05ad03d6 100644 --- a/eden/mononoke/edenapi_service/src/middleware/rate_limiter.rs +++ b/eden/mononoke/edenapi_service/src/middleware/rate_limiter.rs @@ -70,11 +70,14 @@ impl Middleware for ThrottleMiddleware { })?; let identities = state.try_borrow::()?.metadata().identities(); + let metadata = state.try_borrow::()?.metadata(); + let atlas = metadata.clientinfo_atlas(); let limit = rate_limiter.find_rate_limit( Metric::EdenApiQps, Some(identities.clone()), Some(&client_main_id), + atlas, )?; let enforced = match limit.body.raw_config.status { diff --git a/eden/mononoke/git_server/src/middleware/request/ratelimiting.rs b/eden/mononoke/git_server/src/middleware/request/ratelimiting.rs index db323121ae6f9..ddbe1379571eb 100644 --- a/eden/mononoke/git_server/src/middleware/request/ratelimiting.rs +++ b/eden/mononoke/git_server/src/middleware/request/ratelimiting.rs @@ -67,11 +67,13 @@ impl Middleware for UploadPackRateLimitingMiddleware { .client_info() .and_then(|client_info| client_info.request_info.clone()) .and_then(|request_info| request_info.main_id); + let atlas = metadata.clientinfo_atlas(); let mut scuba = self.scuba.clone(); if let LoadShedResult::Fail(err) = rate_limiter.check_load_shed( metadata.identities(), main_client_id.as_deref(), &mut scuba, + atlas, ) { MononokeGitScubaHandler::log_rejected( scuba, diff --git a/eden/mononoke/lfs_server/src/service/middleware.rs b/eden/mononoke/lfs_server/src/service/middleware.rs index 1356f306fe0dd..84e8aa4c25a05 100644 --- a/eden/mononoke/lfs_server/src/service/middleware.rs +++ b/eden/mononoke/lfs_server/src/service/middleware.rs @@ -59,15 +59,20 @@ impl Middleware for ThrottleMiddleware { } } - let (identities, main_client_id) = match state.try_borrow::() { + let (identities, main_client_id, atlas) = match state.try_borrow::() { Some(metadata_state) => { let client_id = metadata_state .metadata() .client_request_info() .and_then(|info| info.main_id.clone()); - (Some(metadata_state.metadata().identities()), client_id) + let atlas = metadata_state.metadata().clientinfo_atlas(); + ( + Some(metadata_state.metadata().identities()), + client_id, + atlas, + ) } - None => (None, None), + None => (None, None, None), }; for limit in self.handle.get().loadshedding_limits().iter() { @@ -77,6 +82,7 @@ impl Middleware for ThrottleMiddleware { main_client_id.as_deref(), &mut self.scuba, OdsCounterManager::new(self.fb), + atlas, ) { let err = HttpError::e429(err); diff --git a/eden/mononoke/rate_limiting/src/config.rs b/eden/mononoke/rate_limiting/src/config.rs index 46f757f43b8b7..748070e182a5a 100644 --- a/eden/mononoke/rate_limiting/src/config.rs +++ b/eden/mononoke/rate_limiting/src/config.rs @@ -17,6 +17,7 @@ use serde::Deserialize; use serde::de::Deserializer; use serde::de::Error as _; +use crate::AtlasTarget; use crate::FciMetric; use crate::LoadShedLimit; use crate::Metric; @@ -49,6 +50,7 @@ impl TryFrom for Target { .map(|s| MononokeIdentity::from_str(&s)) .collect::>()?, )), + rate_limiting_config::Target::atlas(_) => Ok(Target::Atlas(AtlasTarget {})), _ => Err(anyhow!( "Invalid target. Are you using deprecated `and`, `or` or `not` targets?" )), diff --git a/eden/mononoke/rate_limiting/src/facebook.rs b/eden/mononoke/rate_limiting/src/facebook.rs index 3677bb66fd7cd..a45a9f581da81 100644 --- a/eden/mononoke/rate_limiting/src/facebook.rs +++ b/eden/mononoke/rate_limiting/src/facebook.rs @@ -115,6 +115,7 @@ impl RateLimiter for MononokeRateLimits { identities: &MononokeIdentitySet, main_id: Option<&str>, scuba: &mut MononokeScubaSampleBuilder, + atlas: Option, ) -> Result { for limit in &self.config.rate_limits { let fci_metric = limit.fci_metric; @@ -123,7 +124,7 @@ impl RateLimiter for MononokeRateLimits { continue; } - if !limit.applies_to_client(identities, main_id) { + if !limit.applies_to_client(identities, main_id, atlas) { continue; } @@ -155,6 +156,7 @@ impl RateLimiter for MononokeRateLimits { identities: &MononokeIdentitySet, main_id: Option<&str>, scuba: &mut MononokeScubaSampleBuilder, + atlas: Option, ) -> LoadShedResult { for limit in &self.config.load_shed_limits { if let LoadShedResult::Fail(reason) = limit.should_load_shed( @@ -163,6 +165,7 @@ impl RateLimiter for MononokeRateLimits { main_id, scuba, self.ods_counters.clone(), + atlas, ) { return LoadShedResult::Fail(reason); } @@ -188,6 +191,7 @@ impl RateLimiter for MononokeRateLimits { metric: Metric, identities: Option, main_id: Option<&str>, + atlas: Option, ) -> Option { // First, try to find a rate limit that matches the main client ID if let Some(main_id) = main_id { @@ -218,7 +222,7 @@ impl RateLimiter for MononokeRateLimits { .rate_limits .iter() .filter(|r| r.fci_metric.metric == metric) - .filter(|r| r.applies_to_client(&identities, None)) + .filter(|r| r.applies_to_client(&identities, None, atlas)) .for_each(|r| match &r.target { Some(crate::Target::Identities(is)) => { let num_identities = is.len(); diff --git a/eden/mononoke/rate_limiting/src/lib.rs b/eden/mononoke/rate_limiting/src/lib.rs index 79f61fef22009..511a7c6a3ccde 100644 --- a/eden/mononoke/rate_limiting/src/lib.rs +++ b/eden/mononoke/rate_limiting/src/lib.rs @@ -59,6 +59,7 @@ pub trait RateLimiter { identities: &MononokeIdentitySet, main_id: Option<&str>, scuba: &mut MononokeScubaSampleBuilder, + atlas: Option, ) -> Result; fn check_load_shed( @@ -66,6 +67,7 @@ pub trait RateLimiter { identities: &MononokeIdentitySet, main_id: Option<&str>, scuba: &mut MononokeScubaSampleBuilder, + atlas: Option, ) -> LoadShedResult; fn bump_load(&self, metric: Metric, scope: Scope, load: LoadCost); @@ -77,6 +79,7 @@ pub trait RateLimiter { metric: Metric, identities: Option, main_id: Option<&str>, + atlas: Option, ) -> Option; } @@ -192,11 +195,16 @@ pub struct RateLimit { #[cfg(fbcode_build)] impl RateLimit { - fn applies_to_client(&self, identities: &MononokeIdentitySet, main_id: Option<&str>) -> bool { + fn applies_to_client( + &self, + identities: &MononokeIdentitySet, + main_id: Option<&str>, + atlas: Option, + ) -> bool { match &self.target { // TODO (harveyhunt): Pass identities rather than Some(identities) once LFS server has // been updated to require certs. - Some(t) => t.matches_client(Some(identities), main_id), + Some(t) => t.matches_client(Some(identities), main_id, atlas), None => true, } } @@ -245,9 +253,10 @@ impl LoadShedLimit { main_id: Option<&str>, scuba: &mut MononokeScubaSampleBuilder, ods_counters: Arc>, + atlas: Option, ) -> LoadShedResult { let applies_to_client = match &self.target { - Some(t) => t.matches_client(identities, main_id), + Some(t) => t.matches_client(identities, main_id, atlas), None => true, }; @@ -336,6 +345,7 @@ pub enum Target { StaticSlice(StaticSlice), MainClientId(String), Identities(MononokeIdentitySet), + Atlas(AtlasTarget), } #[derive(Debug, Copy, Clone, PartialEq)] @@ -362,6 +372,11 @@ pub struct StaticSlice { target: StaticSliceTarget, } +#[derive(Debug, Clone, PartialEq)] +pub struct AtlasTarget { + // Empty struct - applies to all Atlas environments +} + #[derive(Debug, Clone, PartialEq)] pub enum StaticSliceTarget { Identities(MononokeIdentitySet), @@ -373,6 +388,7 @@ impl Target { &self, identities: Option<&MononokeIdentitySet>, main_client_id: Option<&str>, + atlas: Option, ) -> bool { match self { Self::Identities(target_identities) => { @@ -399,6 +415,10 @@ impl Target { false => false, } } + Self::Atlas(_atlas_target) => { + // Check if the client is an Atlas client + atlas == Some(true) + } } } } @@ -462,28 +482,28 @@ mod test { let ident_target = Target::Identities([test_ident.clone()].into()); - assert!(!ident_target.matches_client(empty_idents.as_ref(), None)); + assert!(!ident_target.matches_client(empty_idents.as_ref(), None, None)); let mut idents = MononokeIdentitySet::new(); idents.insert(test_ident.clone()); idents.insert(test2_ident.clone()); let idents = Some(idents); - assert!(ident_target.matches_client(idents.as_ref(), None)); + assert!(ident_target.matches_client(idents.as_ref(), None, None)); let two_idents = Target::Identities([test_ident, test2_ident].into()); - assert!(two_idents.matches_client(idents.as_ref(), None)); + assert!(two_idents.matches_client(idents.as_ref(), None, None)); let client_id_target = Target::MainClientId(test_client_id.clone()); - assert!(client_id_target.matches_client(None, Some(&test_client_id))); + assert!(client_id_target.matches_client(None, Some(&test_client_id), None)); // Check that all match if the target is empty. let empty_ident_target = Target::Identities([].into()); - assert!(empty_ident_target.matches_client(None, None)); - assert!(empty_ident_target.matches_client(idents.as_ref(), None)); - assert!(empty_ident_target.matches_client(None, Some(&test_client_id))); - assert!(empty_ident_target.matches_client(idents.as_ref(), Some(&test_client_id))); + assert!(empty_ident_target.matches_client(None, None, None)); + assert!(empty_ident_target.matches_client(idents.as_ref(), None, None)); + assert!(empty_ident_target.matches_client(None, Some(&test_client_id), None)); + assert!(empty_ident_target.matches_client(idents.as_ref(), Some(&test_client_id), None)); } #[mononoke::test] @@ -519,6 +539,19 @@ mod test { )); } + #[mononoke::test] + fn test_atlas_target_matches() { + // Test Atlas target - matches all Atlas + let atlas_target = Target::Atlas(AtlasTarget {}); + + // Should match any Atlas client + assert!(atlas_target.matches_client(None, None, Some(true))); + + // Should not match non-Atlas clients + assert!(!atlas_target.matches_client(None, None, Some(false))); + assert!(!atlas_target.matches_client(None, None, None)); + } + #[cfg(fbcode_build)] #[mononoke::test] fn test_static_slice_of_identity_set() { @@ -552,19 +585,19 @@ mod test { let idents2 = Some(idents); // All of SERVICE_IDENTITY: bar - assert!(ident_target.matches_client(idents1.as_ref(), None)); + assert!(ident_target.matches_client(idents1.as_ref(), None, None)); // 20% of SERVICE_IDENTITY: bar. ratelimited host - assert!(twenty_pct_service_identity.matches_client(idents1.as_ref(), None)); + assert!(twenty_pct_service_identity.matches_client(idents1.as_ref(), None, None)); // 20% of SERVICE_IDENTITY: bar. not ratelimited host - assert!(!twenty_pct_service_identity.matches_client(idents2.as_ref(), None)); + assert!(!twenty_pct_service_identity.matches_client(idents2.as_ref(), None, None)); // 100% of SERVICE_IDENTITY: bar - assert!(hundred_pct_service_identity.matches_client(idents1.as_ref(), None)); + assert!(hundred_pct_service_identity.matches_client(idents1.as_ref(), None, None)); // 100% of SERVICE_IDENTITY: bar - assert!(hundred_pct_service_identity.matches_client(idents2.as_ref(), None)); + assert!(hundred_pct_service_identity.matches_client(idents2.as_ref(), None, None)); } #[cfg(fbcode_build)] @@ -623,25 +656,31 @@ mod test { rate_limiter.find_rate_limit( Metric::EgressBytes, Some(idents.clone()), - Some("non_matching_id") + Some("non_matching_id"), + None, ) == Some(empty_target_rate_limit) ); assert!( rate_limiter.find_rate_limit( Metric::EgressBytes, Some(idents.clone()), - Some("client_id") + Some("client_id"), + None, ) == Some(main_client_id_rate_limit.clone()) ); idents.insert(MononokeIdentity::new("TIER", "foo")); assert!( - rate_limiter.find_rate_limit(Metric::EgressBytes, Some(idents.clone()), None) + rate_limiter.find_rate_limit(Metric::EgressBytes, Some(idents.clone()), None, None,) == Some(identities_rate_limit) ); assert!( - rate_limiter.find_rate_limit(Metric::EgressBytes, Some(idents), Some("client_id")) - == Some(main_client_id_rate_limit) + rate_limiter.find_rate_limit( + Metric::EgressBytes, + Some(idents), + Some("client_id"), + None, + ) == Some(main_client_id_rate_limit) ); } } diff --git a/eden/mononoke/rate_limiting/src/oss.rs b/eden/mononoke/rate_limiting/src/oss.rs index 39f9ea9638c16..3a86c3a442860 100644 --- a/eden/mononoke/rate_limiting/src/oss.rs +++ b/eden/mononoke/rate_limiting/src/oss.rs @@ -52,6 +52,7 @@ impl RateLimiter for FakeLimiter { _identities: &MononokeIdentitySet, _main_id: Option<&str>, _scuba: &mut MononokeScubaSampleBuilder, + _atlas: Option, ) -> Result { Ok(RateLimitResult::Pass) } @@ -61,6 +62,7 @@ impl RateLimiter for FakeLimiter { _identities: &MononokeIdentitySet, _main_id: Option<&str>, _scuba: &mut MononokeScubaSampleBuilder, + _atlas: Option, ) -> LoadShedResult { LoadShedResult::Pass } @@ -76,6 +78,7 @@ impl RateLimiter for FakeLimiter { _metric: Metric, _identities: Option, _main_id: Option<&str>, + _atlas: Option, ) -> Option { None } diff --git a/eden/mononoke/scs/if/source_control.thrift b/eden/mononoke/scs/if/source_control.thrift index 3e79a44cb394c..b2257546e4799 100644 --- a/eden/mononoke/scs/if/source_control.thrift +++ b/eden/mononoke/scs/if/source_control.thrift @@ -1148,11 +1148,6 @@ struct RepoDeleteBookmarkParams { 3: optional string service_identity; } -enum CrossRepoPushSource { - NATIVE_TO_THIS_REPO = 0, - PUSH_REDIRECTED = 1, -} - enum BookmarkKindRestrictions { ANY_KIND = 0, ONLY_SCRATCH = 1, diff --git a/eden/mononoke/scs/scs_methods/src/from_request.rs b/eden/mononoke/scs/scs_methods/src/from_request.rs index 11915d8b29bca..2690171f0c8f2 100644 --- a/eden/mononoke/scs/scs_methods/src/from_request.rs +++ b/eden/mononoke/scs/scs_methods/src/from_request.rs @@ -21,7 +21,6 @@ use chrono::TimeZone; use derived_data_manager::DerivableType; use ephemeral_blobstore::BubbleId; use faster_hex::hex_string; -use hooks::CrossRepoPushSource; use metaconfig_types::CommitIdentityScheme; use mononoke_api::BookmarkKey; use mononoke_api::CandidateSelectionHintArgs; @@ -68,21 +67,6 @@ impl FromRequest for BookmarkKey { } } -impl FromRequest for CrossRepoPushSource { - fn from_request( - push_source: &thrift::CrossRepoPushSource, - ) -> Result { - match push_source { - &thrift::CrossRepoPushSource::NATIVE_TO_THIS_REPO => Ok(Self::NativeToThisRepo), - &thrift::CrossRepoPushSource::PUSH_REDIRECTED => Ok(Self::PushRedirected), - other => Err(scs_errors::invalid_request(format!( - "Unknown CrossRepoPushSource: {}", - other - ))), - } - } -} - impl FromRequest for BookmarkKindRestrictions { fn from_request( push_source: &thrift::BookmarkKindRestrictions, diff --git a/eden/mononoke/server/context/src/session.rs b/eden/mononoke/server/context/src/session.rs index 13ab175422b8b..2ffdf70063ae5 100644 --- a/eden/mononoke/server/context/src/session.rs +++ b/eden/mononoke/server/context/src/session.rs @@ -125,10 +125,12 @@ impl SessionContainer { .client_info() .and_then(|client_info| client_info.request_info.clone()) .and_then(|request_info| request_info.main_id); + let atlas = self.metadata().clientinfo_atlas(); match limiter.check_load_shed( self.metadata().identities(), main_client_id.as_deref(), scuba, + atlas, ) { LoadShedResult::Fail(reason) => Err(reason), LoadShedResult::Pass => Ok(()), @@ -150,12 +152,14 @@ impl SessionContainer { .client_info() .and_then(|client_info| client_info.request_info.clone()) .and_then(|request_info| request_info.main_id); + let atlas = self.metadata().clientinfo_atlas(); match limiter .check_rate_limit( metric, self.metadata().identities(), main_client_id.as_deref(), scuba, + atlas, ) .await .unwrap_or(RateLimitResult::Pass) diff --git a/eden/mononoke/server/repo_listener/src/request_handler.rs b/eden/mononoke/server/repo_listener/src/request_handler.rs index 04e88caa24d56..b6e0c555ac682 100644 --- a/eden/mononoke/server/repo_listener/src/request_handler.rs +++ b/eden/mononoke/server/repo_listener/src/request_handler.rs @@ -108,10 +108,12 @@ pub async fn request_handler( .client_info() .and_then(|client_info| client_info.request_info.clone()) .and_then(|request_info| request_info.main_id); + let atlas = metadata.clientinfo_atlas(); rate_limiter.check_load_shed( metadata.identities(), main_client_id.as_deref(), &mut scuba, + atlas, ) } { scuba.log_with_msg("Request rejected due to load shedding", format!("{}", err)); diff --git a/eden/mononoke/tests/integration/server/test-multiplexed-packblob.t b/eden/mononoke/tests/integration/server/test-multiplexed-packblob.t index 2ec77bdcb8027..2fe6383103f60 100644 --- a/eden/mononoke/tests/integration/server/test-multiplexed-packblob.t +++ b/eden/mononoke/tests/integration/server/test-multiplexed-packblob.t @@ -6,28 +6,22 @@ $ . "${TEST_FIXTURES}/library.sh" -setup configuration - $ MULTIPLEXED=1 PACK_BLOB=0 setup_common_config "blob_files" - $ cd "$TESTTMP" - $ testtool_drawdag -R repo <<'EOF' - > C - > | - > B - > | - > A - > # modify: C "C" pad=10000 "C" - > # modify: B "B" pad=10000 "B" - > # modify: A "A" pad=10000 "A" - > EOF - A=b7150acb604b51b1d5786f0f7ea951d8ffe902b7313bdeba9620b2ead15e7d66 - B=fab9d974bdf31ffcc908fc5375502d4d38467b091144512700bd8b023982ab57 - C=c2351942da4871c51880dc56e8371474a4f0e82884c62c45581781daede54887 +setup configuration in usual uncompressed way + $ MULTIPLEXED=1 PACK_BLOB=0 default_setup_blobimport "blob_files" + hg repo + o C [draft;rev=2;26805aba1e60] + │ + o B [draft;rev=1;112478962961] + │ + o A [draft;rev=0;426bada5c675] + $ + blobimporting Check the stores have expected counts $ ls blobstore/0/blobs/ | wc -l - 21 + 33 $ ls blobstore/1/blobs/ | wc -l - 21 + 33 Check that the packed sizes are larger due to the packblob wrappers on store 0 $ PACKED=$(du -s --bytes blobstore/0/blobs/ | cut -f1); UNPACKED=$(du -s --bytes blobstore/1/blobs/ | cut -f1) @@ -37,38 +31,8 @@ Move the uncompressed packed store aside $ mv "$TESTTMP/blobstore/0" "$TESTTMP/blobstore.raw" $ rm -rf "$TESTTMP/blobstore_sync_queue/sqlite_dbs" "$TESTTMP/blobstore" -Import again with zstd compression - $ MULTIPLEXED=1 PACK_BLOB=1 setup_common_config "blob_files" - $ cd "$TESTTMP" - $ cat > mononoke-config/common/storage.toml < # Start new config - > [blobstore.metadata.local] - > local_db_path = "$TESTTMP/monsql" - > [blobstore.blobstore.multiplexed_wal] - > multiplex_id = 1 - > queue_db = { local = { local_db_path = "$TESTTMP/blobstore_sync_queue" } } - > write_quorum = 1 - > multiplex_scuba_table = "file://$TESTTMP/blobstore_trace_scuba.json" - > components = [ - > { blobstore_id = 0, blobstore = { pack = { blobstore = { blob_files = { path = "$TESTTMP/blobstore/0" } }, pack_config = { put_format = { ZstdIndividual = { compression_level = 3 } } } } } }, - > { blobstore_id = 1, blobstore = { pack = { blobstore = { blob_files = { path = "$TESTTMP/blobstore/1" } }, pack_config = { put_format = { ZstdIndividual = { compression_level = 3 } } } } } }, - > ] - > [blobstore.mutable_blobstore] - > blob_files = { path = "$TESTTMP/blobstore/mutable" } - > EOF - $ testtool_drawdag -R repo <<'EOF' - > C - > | - > B - > | - > A - > # modify: C "C" pad=10000 "C" - > # modify: B "B" pad=10000 "B" - > # modify: A "A" pad=10000 "A" - > EOF - A=b7150acb604b51b1d5786f0f7ea951d8ffe902b7313bdeba9620b2ead15e7d66 - B=fab9d974bdf31ffcc908fc5375502d4d38467b091144512700bd8b023982ab57 - C=c2351942da4871c51880dc56e8371474a4f0e82884c62c45581781daede54887 +Blobimport again, but this time enable zstd compression + $ blobimport repo/.hg repo --blobstore-write-zstd=true --blobstore-write-zstd-level=0 Check that the packed sizes are smaller due to compression $ PACKED=$(du -s --bytes blobstore/0/blobs/ | cut -f1); OLDPACKED=$(du -s --bytes blobstore.raw/blobs/ | cut -f1) diff --git a/eden/mononoke/tests/integration/server/test-server-init.t b/eden/mononoke/tests/integration/server/test-server-init.t index 4f77d01e4400b..ac9d6d765a898 100644 --- a/eden/mononoke/tests/integration/server/test-server-init.t +++ b/eden/mononoke/tests/integration/server/test-server-init.t @@ -23,19 +23,69 @@ setup repo $ hginit_treemanifest repo $ cd repo - - $ touch a - $ hg add a - $ hg ci -ma - $ hg log - commit: 3903775176ed - user: test - date: Thu Jan 01 00:00:00 1970 +0000 - summary: a - (re) - $ hg book master_bookmark + $ testtool_drawdag -R repo --no-default-files <<'EOF' + > a-b-c-newdir-modifyfile-rename + > # modify: a "a" "\n" + > # modify: b "a" "\n" + > # modify: b "b" "\n" + > # modify: c "a" "\n" + > # modify: c "b" "\n" + > # modify: c "c" "content\n" + > # modify: newdir "a" "\n" + > # modify: newdir "b" "\n" + > # modify: newdir "c" "content\n" + > # modify: newdir "dir/1" "1\n" + > # modify: newdir "dir/2" "2\n" + > # modify: modifyfile "a" "\n" + > # modify: modifyfile "b" "\n" + > # modify: modifyfile "c" "cc\n" + > # modify: modifyfile "dir/1" "1\n" + > # modify: modifyfile "dir/2" "2\n" + > # copy: rename "dir/rename" "1\n" modifyfile "dir/1" + > # delete: rename "dir/1" + > # modify: rename "a" "\n" + > # modify: rename "b" "\n" + > # modify: rename "c" "cc\n" + > # modify: rename "dir/2" "2\n" + > # message: a "a" + > # message: b "b" + > # message: c "c" + > # message: newdir "new directory" + > # message: modifyfile "modify file" + > # message: rename "rename" + > # bookmark: rename master_bookmark2 + > EOF + a=6a2ad26f394ca4d270bea9aa4ef57731a99a46057eb790789ff4eee79a7ba5f3 + b=a813a976d95b5674a7ccdf34c815a5a136ba67b9437feeccbefb4387ca3acefb + c=f1fa7c6ecbeb4497745f10de2fb3c37d424f6ae3a0e204f5ac48aca61ea8ed9a + modifyfile=f0f10d3a94843d1dcbcd170b4ead2e5da946d16020befc2a1549aa97d0ab32cc + newdir=62651ea03b71d00067a442c8c11490eaaee62d81bfa4aeacb831be3d2c4abd7d + rename=c6c8fe2b4788a2c4f851f333cf6bda1854622643db48960b6406eda3116d392e + + $ testtool_drawdag -R repo --no-default-files <<'EOF' + > A-B-D + > A-C-D + > # modify: A "D" "x\n" + > # modify: B "D" "1\n" + > # modify: C "D" "2\n" + > # modify: D "D" "1\n2\n" + > # message: A "A" + > # message: B "B" + > # message: C "C" + > # message: D "D" + > # bookmark: D master_bookmark + > EOF + A=c28cb6ca9425c818e0542de681d71608239dff255c35f7c00221e442d78606f2 + B=3554e5586aa62f5b9b6ce24ae0a0adcfb91eb253058eea80795b189602abbb4d + C=722771c3b789544faf7a86626709ddd6fa0c450452cf6480441dfd1661aeddf4 + D=e75cd16fce5a00c2722a6d53bc57641a7bb36560d0978123d93f41fcd1b7a452 $ cd $TESTTMP +start mononoke + + $ mononoke + $ wait_for_mononoke + setup repo2 $ cat >> $HGRCPATH < [extensions] @@ -44,72 +94,7 @@ setup repo2 > cachepath=$TESTTMP/cachepath > EOF $ hg clone -q mono:repo repo2 --noupdate - $ cd repo2 - $ hg pull -q - $ cd $TESTTMP - $ cd repo - $ touch b - $ hg add b - $ hg ci -mb - $ echo content > c - $ hg add c - $ hg ci -mc - $ mkdir dir - $ echo 1 > dir/1 - $ mkdir dir2 - $ echo 2 > dir/2 - $ hg addremove - adding dir/1 - adding dir/2 - $ hg ci -m 'new directory' - $ echo cc > c - $ hg addremove - $ hg ci -m 'modify file' - $ hg mv dir/1 dir/rename - $ hg ci -m 'rename' - $ hg debugdrawdag <<'EOS' - > D # D/D=1\n2\n - > /| # B/D=1\n - > B C # C/D=2\n - > |/ # A/D=x\n - > A - > EOS - $ hg log --graph -T '{node|short} {desc}' - o e635b24c95f7 D - ├─╮ - │ o d351044ef463 C - │ │ - o │ 9a827afb7e25 B - ├─╯ - o af6aa0dfdf3d A - (re) - @ 9f8e7242d9fa rename - │ - o 586ef37a04f7 modify file - │ - o e343d2f326cf new directory - │ - o 3e19bf519e9a c - │ - o 0e067c57feba b - │ - o 3903775176ed a - (re) - -setup master bookmarks - - $ hg bookmark master_bookmark -r e635b24c95f7 -f - $ hg bookmark master_bookmark2 -r 9f8e7242d9fa - -blobimport - - $ cd .. - $ blobimport repo/.hg repo - -start mononoke - - $ start_and_wait_for_mononoke_server $ hg debugwireargs mono:disabled_repo one two --three three remote: Unknown Repo: remote: Error: @@ -120,26 +105,47 @@ start mononoke one two three None None $ cd repo2 - $ hg up -q "min(all())" -Test a pull of one specific revision - $ hg pull -r 3e19bf519e9af6c66edf28380101a92122cbea50 -q -Pull the rest $ hg pull -q + $ HG_C_HASH=$(hg log -r 'desc(C) and ancestors(master_bookmark)' -T '{node}') + $ cd $TESTTMP - $ hg log -r '3903775176ed::586ef37a04f7' --graph -T '{node|short} {desc}' - o 586ef37a04f7 modify file +Create repo3 to test pull by specific hash + $ hg clone -q mono:repo repo3 --noupdate + $ cd repo3 + $ hg up -q "min(all())" +Test a pull of one specific revision by hash + $ hg pull -r $HG_C_HASH -q + $ hg log -r $HG_C_HASH -T '{desc}\n' + C + + $ cd ../repo2 + +Verify the merge DAG was created correctly + $ hg log -r 'ancestors(master_bookmark)' --graph -T '{node|short} {desc}' + o 028746d393f5 D + ├─╮ + │ o b0d80666c945 C + │ │ + o │ 40fd45cc2dd2 B + ├─╯ + o 7459d653bda7 A + (re) + + $ hg log -r 'ancestors(master_bookmark2)' --graph -T '{node|short} {desc}' + o 280cea29404f rename │ - o e343d2f326cf new directory + o a1af22172c7e modify file │ - o 3e19bf519e9a c + o 34ec82955987 new directory │ - o 0e067c57feba b + o 635623e45e72 c │ - @ 3903775176ed a + o c79659601245 b + │ + o 859ac8b08f08 a (re) $ ls - a - $ hg up 9f8e7242d9fa -q + $ hg up master_bookmark2 -q $ ls a b @@ -147,31 +153,31 @@ Pull the rest dir $ cat c cc - $ hg up 9f8e7242d9fa -q + $ hg up master_bookmark2 -q $ hg log c -T '{node|short} {desc}\n' warning: file log can be slow on large repos - use -f to speed it up - 586ef37a04f7 modify file - 3e19bf519e9a c + a1af22172c7e modify file + 635623e45e72 c $ cat dir/rename 1 $ cat dir/2 2 $ hg log dir/rename -f -T '{node|short} {desc}\n' - 9f8e7242d9fa rename - e343d2f326cf new directory - $ hg st --change 9f8e7242d9fa -C + 280cea29404f rename + 34ec82955987 new directory + $ hg st --change master_bookmark2 -C A dir/rename dir/1 R dir/1 - $ hg up -q e635b24c95f7 + $ hg up -q master_bookmark Sort the output because it may be unpredictable because of the merge $ hg log D --follow -T '{node|short} {desc}\n' | sort - 9a827afb7e25 B - af6aa0dfdf3d A - d351044ef463 C - e635b24c95f7 D + 028746d393f5 D + 40fd45cc2dd2 B + 7459d653bda7 A + b0d80666c945 C Create a new bookmark and try and send it over the wire Test commented while we have no bookmark support in blobimport or easy method diff --git a/eden/mononoke/tests/integration/server/test-server-new-repo.t b/eden/mononoke/tests/integration/server/test-server-new-repo.t index cfe2fc2cde9c1..e8ea08a2e6645 100644 --- a/eden/mononoke/tests/integration/server/test-server-new-repo.t +++ b/eden/mononoke/tests/integration/server/test-server-new-repo.t @@ -13,15 +13,15 @@ setup configuration $ setup_common_config $ cd $TESTTMP +setup repo + $ testtool_drawdag -R repo < A + > # bookmark: A master_bookmark + > EOF + A=aa53d24251ff3f54b1b2c29ae02826701b2abeb0079f1bb13b8434b54cd87675 + start mononoke $ start_and_wait_for_mononoke_server -setup repo - $ hg clone -q mono:repo repo - $ cd repo - $ echo "a file content" > a - $ hg add a - $ hg ci -ma - $ hg push -q --to master_bookmark --create clone from the new repo as well $ hg clone -q mono:repo repo-clone @@ -31,9 +31,9 @@ Push with bookmark $ echo withbook > withbook && hg addremove && hg ci -m withbook adding withbook $ hg push --to withbook --create - pushing rev 11f53bbd855a to destination mono:repo bookmark withbook + pushing rev cdbb2b8b2cf1 to destination mono:repo bookmark withbook searching for changes exporting bookmark withbook $ hg book --remote - remote/master_bookmark 0e7ec5675652a04069cbf976a42e45b740f3243c - remote/withbook 11f53bbd855ac06521a8895bd57e6ce5f46a9980 + remote/master_bookmark 20ca2a4749a439b459125ef0f6a4f26e88ee7538 + remote/withbook cdbb2b8b2cf1612cd6a1271c96a7a89d98b36dd4 diff --git a/eden/mononoke/tests/integration/test-blobstore-healer.t b/eden/mononoke/tests/integration/test-blobstore-healer.t index d59ea1158d94e..be0be355e81ff 100644 --- a/eden/mononoke/tests/integration/test-blobstore-healer.t +++ b/eden/mononoke/tests/integration/test-blobstore-healer.t @@ -8,37 +8,42 @@ $ . "${TEST_FIXTURES}/library.sh" setup configuration - $ MULTIPLEXED=2 default_setup_blobimport "blob_files" - hg repo - o C [draft;rev=2;26805aba1e60] - │ - o B [draft;rev=1;112478962961] - │ - o A [draft;rev=0;426bada5c675] - $ - blobimporting + $ MULTIPLEXED=2 setup_common_config "blob_files" + $ cd "$TESTTMP" + +Create repo using testtool + $ testtool_drawdag -R repo < C + > | + > B + > | + > A + > EOF + A=aa53d24251ff3f54b1b2c29ae02826701b2abeb0079f1bb13b8434b54cd87675 + B=f8c75e41a0c4d29281df765f39de47bca1dcadfdc55ada4ccc2f6df567201658 + C=e32a1e342cdb1e38e88466b4c1a01ae9f410024017aa21dc0a1c5da6b3963bf2 Check that healer queue has drained $ read_blobstore_wal_queue_size 0 -Erase the sqllites and blobstore_sync_queue - $ rm -rf "$TESTTMP/blobstore" - -blobimport them into Mononoke storage again, but with write failures on one side - $ blobimport repo/.hg repo --blobstore-write-chaos-rate=1 +Populate WAL queue by simulating write failures - delete blobs from blobstore 0 and populate WAL + $ mononoke_testtool populate-wal -R repo --blobstore-path "$TESTTMP/blobstore" --source-blobstore-id 1 --delete-target-blobs --storage-id=blobstore + Found 21 blobs in source blobstore 1 + Deleted 21 blobs from target blobstore 0 + Inserted 21 WAL entries for target multiplex_id 1 Check the stores have expected counts $ ls blobstore/0/blobs/ | wc -l 0 $ ls blobstore/1/blobs/ | wc -l - 33 + 21 $ ls blobstore/2/blobs/ | wc -l - 33 + 21 Check that healer queue has successful items $ read_blobstore_wal_queue_size - 33 + 21 Run the heal, with write errors injected, simulating store still bad $ function count_log() { @@ -46,8 +51,8 @@ Run the heal, with write errors injected, simulating store still bad > uniq -c | sed 's/^ *//' > } $ mononoke_blobstore_healer --blobstore-write-chaos-rate 1 -q --iteration-limit=1 --heal-min-age-secs=0 --storage-id=blobstore --sync-queue-limit=100 2>&1 | count_log | grep -v "speed" | grep -E -v "^1 (Monitoring|Discovered) regions:.*" - 1 [INFO] Fetched 33 distinct put operations - 1 [INFO] Found 33 blobs to be healed... Doing it with weight limit 10000000000, max concurrency: 100 + 1 [INFO] Fetched 21 distinct put operations + 1 [INFO] Found 21 blobs to be healed... Doing it with weight limit 10000000000, max concurrency: 100 1 [INFO] Couldn't heal blob repo0000.alias.gitsha1.7371f47a6f8bd23a8fa1a8b2a9479cdd76380e54 in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.alias.gitsha1.8c7e5a667f1b771847fe88c01c3de34413a1b220 in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.alias.gitsha1.96d80cd6c4e7158dbebd0849f4fb7ce513e5828c in these blobstores: {BlobstoreId(0)} @@ -60,43 +65,31 @@ Run the heal, with write errors injected, simulating store still bad 1 [INFO] Couldn't heal blob repo0000.alias.sha256.559aead08264d5795d3909718cdd05abd49572e84fe55590eef31a88a08fdffd in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.alias.sha256.6b23c0d5f35d1b11f9b683f0b0a617355deb11277d91ae091d399c655b87940d in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.alias.sha256.df7e70e5021544f4834bbee64a9e3789febc4be81470df629cad6ddb03320a5c in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.459f16ae564c501cb408c1e5b60fc98a1e8b8e97b9409c7520658bfa1577fb66 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.9feb8ddd3e8eddcfa3a4913b57df7842bedf84b8ea3b7b3fcb14c6424aa81fec in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.c3384961b16276f2db77df9d7c874bbe981cf0525bd6f84a502f919044f2dabd in these blobstores: {BlobstoreId(0)} + 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.aa53d24251ff3f54b1b2c29ae02826701b2abeb0079f1bb13b8434b54cd87675 in these blobstores: {BlobstoreId(0)} + 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.e32a1e342cdb1e38e88466b4c1a01ae9f410024017aa21dc0a1c5da6b3963bf2 in these blobstores: {BlobstoreId(0)} + 1 [INFO] Couldn't heal blob repo0000.changeset.blake2.f8c75e41a0c4d29281df765f39de47bca1dcadfdc55ada4ccc2f6df567201658 in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content.blake2.55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content.blake2.896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content.blake2.eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9 in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content_metadata2.blake2.55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content_metadata2.blake2.896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d in these blobstores: {BlobstoreId(0)} 1 [INFO] Couldn't heal blob repo0000.content_metadata2.blake2.eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.filenode_lookup.61585a6b75335f6ec9540101b7147908564f2699dcad59134fdf23cb086787ad in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.filenode_lookup.9915e555ad3fed014aa36a4e48549c1130fddffc7660589f42af5f0520f1118e in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.filenode_lookup.a0377040953a1a3762b7c59cb526797c1afd7ae6fcebb4d11e3c9186a56edb4e in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgchangeset.sha1.112478962961147124edd43549aedd1a335e44bf in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgchangeset.sha1.26805aba1e600a82e93661149f2313866a221a7b in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgchangeset.sha1.426bada5c67598ca65036d57d9e4b64b0c1ce7a0 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgfilenode.sha1.005d992c5dcf32993668f7cede29d296c494a5d9 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgfilenode.sha1.35e7525ce3a48913275d7061dd9a867ffef1e34d in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgfilenode.sha1.a2e456504a5e61f763f1a0b36a6c247c7541b2b3 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgmanifest.sha1.41b34f08c1356f6ad068e9ab9b43d984245111aa in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgmanifest.sha1.7c9b4fd8b49377e2fead2e9610bb8db910a98c53 in these blobstores: {BlobstoreId(0)} - 1 [INFO] Couldn't heal blob repo0000.hgmanifest.sha1.eb79886383871977bccdb3000c275a279f0d4c99 in these blobstores: {BlobstoreId(0)} - 1 [INFO] For 33 processed entries and 33 blobstore keys: healthy blobs 0, healed blobs 0, failed to heal 33, missing blobs 0 - 1 [INFO] Requeuing 33 queue entries for another healing attempt - 1 [INFO] Deleting 33 actioned queue entries + 1 [INFO] For 21 processed entries and 21 blobstore keys: healthy blobs 0, healed blobs 0, failed to heal 21, missing blobs 0 + 1 [INFO] Requeuing 21 queue entries for another healing attempt + 1 [INFO] Deleting 21 actioned queue entries 1 [INFO] Iteration rows processed: * rows, *s; total: * rows, *s (glob) 1 [INFO] The last batch was not full size, waiting... Check that healer queue still has the items, should not have drained $ read_blobstore_wal_queue_size - 33 + 21 Healer run again now store recovered $ mononoke_blobstore_healer -q --iteration-limit=1 --heal-min-age-secs=0 --storage-id=blobstore --sync-queue-limit=100 2>&1 | count_log | grep -E -v "^1 (Monitoring|Discovered) regions:.*" - 1 [INFO] Fetched 33 distinct put operations - 1 [INFO] Found 33 blobs to be healed... Doing it with weight limit 10000000000, max concurrency: 100 - 1 [INFO] For 33 processed entries and 33 blobstore keys: healthy blobs 0, healed blobs 33, failed to heal 0, missing blobs 0 - 1 [INFO] Deleting 33 actioned queue entries + 1 [INFO] Fetched 21 distinct put operations + 1 [INFO] Found 21 blobs to be healed... Doing it with weight limit 10000000000, max concurrency: 100 + 1 [INFO] For 21 processed entries and 21 blobstore keys: healthy blobs 0, healed blobs 21, failed to heal 0, missing blobs 0 + 1 [INFO] Deleting 21 actioned queue entries 1 [INFO] Iteration rows processed: * rows, *s; total: * rows, *s (glob) 1 [INFO] The last batch was not full size, waiting... @@ -106,8 +99,8 @@ Check that healer queue has drained Check the stores have expected counts $ ls blobstore/0/blobs/ | wc -l - 33 + 21 $ ls blobstore/1/blobs/ | wc -l - 33 + 21 $ ls blobstore/2/blobs/ | wc -l - 33 + 21 diff --git a/eden/mononoke/tools/testtool/BUCK b/eden/mononoke/tools/testtool/BUCK index 342a2dea90125..c61b80f15f242 100644 --- a/eden/mononoke/tools/testtool/BUCK +++ b/eden/mononoke/tools/testtool/BUCK @@ -5,6 +5,10 @@ oncall("scm_server_infra") rust_binary( name = "testtool", srcs = glob(["src/**/*.rs"]), + test_deps = [ + "fbsource//third-party/rust:tempfile", + "//eden/mononoke/mononoke_macros:mononoke_macros", + ], deps = [ "fbsource//third-party/rust:anyhow", "fbsource//third-party/rust:bytes", @@ -25,6 +29,7 @@ rust_binary( "//eden/mononoke/derived_data:derived_data_manager", "//eden/mononoke/derived_data/mercurial_derivation:mercurial_derivation", "//eden/mononoke/features/changesets_creation:changesets_creation", + "//eden/mononoke/metaconfig:metaconfig_types", "//eden/mononoke/mononoke_types:mononoke_types", "//eden/mononoke/repo_attributes/bonsai_hg_mapping:bonsai_hg_mapping", "//eden/mononoke/repo_attributes/bookmarks:bookmarks", diff --git a/eden/mononoke/tools/testtool/Cargo.toml b/eden/mononoke/tools/testtool/Cargo.toml index a015e53019cb9..1904779a004e2 100644 --- a/eden/mononoke/tools/testtool/Cargo.toml +++ b/eden/mononoke/tools/testtool/Cargo.toml @@ -23,6 +23,7 @@ facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust fbinit = { version = "0.2.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } filestore = { version = "0.1.0", path = "../../repo_attributes/filestore" } mercurial_derivation = { version = "0.1.0", path = "../../derived_data/mercurial_derivation" } +metaconfig_types = { version = "0.1.0", path = "../../metaconfig/types" } mononoke_app = { version = "0.1.0", path = "../../cmdlib/mononoke_app" } mononoke_types = { version = "0.1.0", path = "../../mononoke_types" } repo_blobstore = { version = "0.1.0", path = "../../repo_attributes/repo_blobstore" } @@ -37,3 +38,7 @@ sql_construct = { version = "0.1.0", path = "../../common/sql_construct" } tests_utils = { version = "0.1.0", path = "../../tests/utils" } tokio = { version = "1.47.1", features = ["full", "test-util", "tracing"] } topo_sort = { version = "0.1.0", path = "../../common/topo_sort" } + +[dev-dependencies] +mononoke_macros = { version = "0.1.0", path = "../../mononoke_macros" } +tempfile = "3.22" diff --git a/eden/mononoke/tools/testtool/src/commands/populate_wal.rs b/eden/mononoke/tools/testtool/src/commands/populate_wal.rs index d3422c9845f56..4b4a4b5895f3c 100644 --- a/eden/mononoke/tools/testtool/src/commands/populate_wal.rs +++ b/eden/mononoke/tools/testtool/src/commands/populate_wal.rs @@ -26,10 +26,12 @@ use blobstore_sync_queue::BlobstoreWalEntry; use blobstore_sync_queue::SqlBlobstoreWalBuilder; use clap::Parser; use context::CoreContext; +use metaconfig_types::BlobConfig; use mononoke_app::MononokeApp; use mononoke_app::args::RepoArgs; use mononoke_types::Timestamp; -use sql_construct::SqlConstruct; +use repo_identity::RepoIdentityRef; +use sql_construct::SqlConstructFromShardedDatabaseConfig; use crate::repo::Repo; @@ -47,10 +49,6 @@ pub struct CommandArgs { #[clap(long)] source_blobstore_id: u32, - /// Target blobstore ID that is "missing" the blobs (multiplex_id) - #[clap(long)] - target_blobstore_id: i32, - /// Delete blobs from target blobstore directory #[clap(long)] delete_target_blobs: bool, @@ -59,19 +57,26 @@ pub struct CommandArgs { #[clap(long, default_value = "0")] delete_from_blobstore_id: u32, - /// Path to the blobstore sync queue database - #[clap(long, default_value = "")] - wal_db_path: String, + /// Storage ID to get the WAL from (recommended, extracts multiplex_id from config) + #[clap(long)] + storage_id: Option, + + /// Target blobstore ID to use as multiplex_id (legacy, for backward compatibility) + #[clap(long)] + target_blobstore_id: Option, } pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> { let ctx = app.new_basic_context(); - let _repo: Repo = app + let repo: Repo = app .open_repo(&args.repo_args) .await .context("Failed to open repo")?; + // Get repo prefix for blob keys + let repo_prefix = repo.repo_identity().name(); + // Construct paths let source_blobs_dir = args .blobstore_path @@ -83,7 +88,7 @@ pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> { .join("blobs"); // Get list of blob keys from source blobstore - let blob_keys = list_blob_keys(&source_blobs_dir)?; + let blob_keys = list_blob_keys(&source_blobs_dir, repo_prefix)?; println!( "Found {} blobs in source blobstore {}", @@ -108,39 +113,101 @@ pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> { ); } - // Insert WAL entries - let wal_db_path = if args.wal_db_path.is_empty() { - // Default to $TESTTMP/blobstore_sync_queue/sqlite_dbs (the database file) - let testtmp = std::env::var("TESTTMP").context("TESTTMP environment variable not set")?; - PathBuf::from(testtmp) - .join("blobstore_sync_queue") - .join("sqlite_dbs") - } else { - PathBuf::from(&args.wal_db_path) - }; + // Determine which approach to use for getting WAL and multiplex_id + let (wal, multiplex_id) = match (&args.storage_id, args.target_blobstore_id) { + // Preferred approach: Use storage_id to extract multiplex_id from config + (Some(storage_id), _) => { + let storage_configs = app.storage_configs(); + let storage_config = storage_configs + .storage + .get(storage_id) + .ok_or_else(|| anyhow!("Storage config not found for ID: {}", storage_id))?; + + // Extract queue_db and multiplex_id from MultiplexedWal config + let (queue_db, multiplex_id) = match &storage_config.blobstore { + BlobConfig::MultiplexedWal { + queue_db, + multiplex_id, + .. + } => (queue_db, *multiplex_id), + _ => { + return Err(anyhow!( + "Storage config for '{}' is not MultiplexedWal", + storage_id + )); + } + }; - // Create parent directory if it doesn't exist - if let Some(parent) = wal_db_path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!( - "Failed to create WAL database directory: {}", - parent.display() + // Build the WAL using SqlBlobstoreWalBuilder + let wal = SqlBlobstoreWalBuilder::with_sharded_database_config( + ctx.fb, + queue_db, + app.mysql_options(), + false, // readonly_storage ) - })?; - } + .context("While opening WAL")? + .build(ctx.sql_query_telemetry()); + + (wal, multiplex_id.into()) + } + + // Legacy approach: Use target_blobstore_id directly as multiplex_id + (None, Some(target_blobstore_id)) => { + // Get repo config and extract blobstore config directly + let repo_configs = app.repo_configs(); + let (_repo_name, repo_config) = repo_configs + .get_repo_config(repo.repo_identity().id()) + .ok_or_else(|| { + anyhow!( + "Repo config not found for repo_id: {:?}", + repo.repo_identity().id() + ) + })?; + + // Extract queue_db from the repo's MultiplexedWal config + let queue_db = match &repo_config.storage_config.blobstore { + BlobConfig::MultiplexedWal { queue_db, .. } => queue_db, + _ => { + return Err(anyhow!( + "Repo storage config is not MultiplexedWal (required for WAL). Use --storage-id instead." + )); + } + }; + + // Build the WAL using SqlBlobstoreWalBuilder + let wal = SqlBlobstoreWalBuilder::with_sharded_database_config( + ctx.fb, + queue_db, + app.mysql_options(), + false, // readonly_storage + ) + .context("While opening WAL")? + .build(ctx.sql_query_telemetry()); + + // Use target_blobstore_id as multiplex_id (legacy behavior) + (wal, target_blobstore_id as i32) + } + + // Error: Neither parameter specified + (None, None) => { + return Err(anyhow!( + "Must specify either --storage-id (recommended) or --target-blobstore-id (legacy)" + )); + } + }; - insert_wal_entries(&ctx, &wal_db_path, &blob_keys, args.target_blobstore_id).await?; + insert_wal_entries(&ctx, &wal, &blob_keys, multiplex_id).await?; eprintln!( "Inserted {} WAL entries for target multiplex_id {}", blob_keys.len(), - args.target_blobstore_id + multiplex_id ); Ok(()) } -fn list_blob_keys(blobs_dir: &Path) -> Result> { +fn list_blob_keys(blobs_dir: &Path, _repo_prefix: &str) -> Result> { if !blobs_dir.exists() { return Err(anyhow!( "Blobstore directory does not exist: {}", @@ -162,7 +229,7 @@ fn list_blob_keys(blobs_dir: &Path) -> Result> { .and_then(|n| n.to_str()) .ok_or_else(|| anyhow!("Invalid filename"))?; - // Remove "blob-" prefix if present + // Remove "blob-" prefix if present - the rest is the full blobstore key let key = if let Some(stripped) = filename.strip_prefix("blob-") { stripped.to_string() } else { @@ -209,24 +276,254 @@ fn delete_blobs(target_dir: &Path, blob_keys: &[(String, u64)]) -> Result<()> { async fn insert_wal_entries( ctx: &CoreContext, - wal_db_path: &Path, + wal: &dyn BlobstoreWal, blob_keys: &[(String, u64)], multiplex_id: i32, ) -> Result<()> { - // Build WAL using SQLite path - let wal = SqlBlobstoreWalBuilder::with_sqlite_path(wal_db_path, false /* readonly */)? - .build(ctx.sql_query_telemetry()); - let timestamp = Timestamp::now(); - // Insert entries for each blob - for (key, size) in blob_keys { - let entry = BlobstoreWalEntry::new(key.clone(), multiplex_id.into(), timestamp, *size); + // Collect all entries into a vector for batch insertion + let entries: Vec = blob_keys + .iter() + .map(|(key, size)| { + BlobstoreWalEntry::new(key.clone(), multiplex_id.into(), timestamp, *size) + }) + .collect(); - wal.log(ctx, entry) - .await - .with_context(|| format!("Failed to insert WAL entry for key: {}", key))?; - } + // Insert all entries in one batch + wal.log_many(ctx, entries) + .await + .context("Failed to insert WAL entries")?; Ok(()) } + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + + use mononoke_macros::mononoke; + + use super::*; + + #[mononoke::test] + fn test_list_blob_keys_with_prefix() { + let temp_dir = tempfile::tempdir().unwrap(); + let blobs_dir = temp_dir.path().join("blobs"); + fs::create_dir(&blobs_dir).unwrap(); + + // Create test blobs with "blob-" prefix + let test_data = vec![ + ("blob-repo0000.content.blake2.abc123", b"content1"), + ("blob-repo0000.content.blake2.def456", b"content2"), + ]; + + for (name, content) in &test_data { + let mut file = File::create(blobs_dir.join(name)).unwrap(); + file.write_all(*content).unwrap(); + } + + let result = list_blob_keys(&blobs_dir, "repo0000").unwrap(); + + assert_eq!(result.len(), 2); + // Keys should have "blob-" prefix stripped + assert!( + result + .iter() + .any(|(key, size)| key == "repo0000.content.blake2.abc123" && *size == 8) + ); + assert!( + result + .iter() + .any(|(key, size)| key == "repo0000.content.blake2.def456" && *size == 8) + ); + } + + #[mononoke::test] + fn test_list_blob_keys_without_prefix() { + let temp_dir = tempfile::tempdir().unwrap(); + let blobs_dir = temp_dir.path().join("blobs"); + fs::create_dir(&blobs_dir).unwrap(); + + // Create test blobs without "blob-" prefix + let test_data = vec![ + ("repo0000.alias.sha256.xyz789", b"data1"), + ("repo0000.changeset.blake2.aaa111", b"data2"), + ]; + + for (name, content) in &test_data { + let mut file = File::create(blobs_dir.join(name)).unwrap(); + file.write_all(*content).unwrap(); + } + + let result = list_blob_keys(&blobs_dir, "repo0000").unwrap(); + + assert_eq!(result.len(), 2); + assert!( + result + .iter() + .any(|(key, size)| key == "repo0000.alias.sha256.xyz789" && *size == 5) + ); + assert!( + result + .iter() + .any(|(key, size)| key == "repo0000.changeset.blake2.aaa111" && *size == 5) + ); + } + + #[mononoke::test] + fn test_list_blob_keys_empty_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + let blobs_dir = temp_dir.path().join("blobs"); + fs::create_dir(&blobs_dir).unwrap(); + + let result = list_blob_keys(&blobs_dir, "repo0000").unwrap(); + + assert_eq!(result.len(), 0); + } + + #[mononoke::test] + fn test_list_blob_keys_nonexistent_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + let blobs_dir = temp_dir.path().join("nonexistent"); + + let result = list_blob_keys(&blobs_dir, "repo0000"); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Blobstore directory does not exist") + ); + } + + #[mononoke::test] + fn test_delete_blobs_with_prefix() { + let temp_dir = tempfile::tempdir().unwrap(); + let target_dir = temp_dir.path().join("target"); + fs::create_dir(&target_dir).unwrap(); + + // Create test blobs with "blob-" prefix + File::create(target_dir.join("blob-repo0000.content.blake2.abc123")).unwrap(); + File::create(target_dir.join("blob-repo0000.alias.sha256.def456")).unwrap(); + + let blob_keys = vec![ + ("repo0000.content.blake2.abc123".to_string(), 100u64), + ("repo0000.alias.sha256.def456".to_string(), 200u64), + ]; + + delete_blobs(&target_dir, &blob_keys).unwrap(); + + // Files should be deleted + assert!( + !target_dir + .join("blob-repo0000.content.blake2.abc123") + .exists() + ); + assert!( + !target_dir + .join("blob-repo0000.alias.sha256.def456") + .exists() + ); + } + + #[mononoke::test] + fn test_delete_blobs_without_prefix() { + let temp_dir = tempfile::tempdir().unwrap(); + let target_dir = temp_dir.path().join("target"); + fs::create_dir(&target_dir).unwrap(); + + // Create test blobs without "blob-" prefix + File::create(target_dir.join("repo0000.content.blake2.xyz789")).unwrap(); + File::create(target_dir.join("repo0000.changeset.blake2.aaa111")).unwrap(); + + let blob_keys = vec![ + ("repo0000.content.blake2.xyz789".to_string(), 100u64), + ("repo0000.changeset.blake2.aaa111".to_string(), 200u64), + ]; + + delete_blobs(&target_dir, &blob_keys).unwrap(); + + // Files should be deleted + assert!(!target_dir.join("repo0000.content.blake2.xyz789").exists()); + assert!(!target_dir.join("repo0000.changeset.blake2.aaa111").exists()); + } + + #[mononoke::test] + fn test_delete_blobs_nonexistent_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + let target_dir = temp_dir.path().join("nonexistent"); + + let blob_keys = vec![("repo0000.content.blake2.abc123".to_string(), 100u64)]; + + // Should not error when directory doesn't exist + let result = delete_blobs(&target_dir, &blob_keys); + assert!(result.is_ok()); + } + + #[mononoke::test] + fn test_delete_blobs_missing_files() { + let temp_dir = tempfile::tempdir().unwrap(); + let target_dir = temp_dir.path().join("target"); + fs::create_dir(&target_dir).unwrap(); + + // Create only one of the files + File::create(target_dir.join("blob-repo0000.content.blake2.abc123")).unwrap(); + + let blob_keys = vec![ + ("repo0000.content.blake2.abc123".to_string(), 100u64), + ("repo0000.nonexistent.blake2.missing".to_string(), 200u64), + ]; + + // Should not error when some files are missing + let result = delete_blobs(&target_dir, &blob_keys); + assert!(result.is_ok()); + + // Existing file should be deleted + assert!( + !target_dir + .join("blob-repo0000.content.blake2.abc123") + .exists() + ); + } + + #[mononoke::test] + fn test_list_and_delete_integration() { + let temp_dir = tempfile::tempdir().unwrap(); + let source_dir = temp_dir.path().join("source"); + let target_dir = temp_dir.path().join("target"); + fs::create_dir(&source_dir).unwrap(); + fs::create_dir(&target_dir).unwrap(); + + // Create identical blobs in both directories + let test_files = vec![ + "blob-repo0000.content.blake2.test1", + "blob-repo0000.content.blake2.test2", + "blob-repo0000.alias.sha256.test3", + ]; + + for filename in &test_files { + File::create(source_dir.join(filename)).unwrap(); + File::create(target_dir.join(filename)).unwrap(); + } + + // List blobs from source + let blob_keys = list_blob_keys(&source_dir, "repo0000").unwrap(); + assert_eq!(blob_keys.len(), 3); + + // Delete from target + delete_blobs(&target_dir, &blob_keys).unwrap(); + + // Target should be empty + for filename in &test_files { + assert!(!target_dir.join(filename).exists()); + } + + // Source should still have files + for filename in &test_files { + assert!(source_dir.join(filename).exists()); + } + } +} diff --git a/eden/scm/lib/third-party/streampager/Cargo.toml b/eden/scm/lib/third-party/streampager/Cargo.toml index 676ec7e0367c4..e4c84723462b0 100644 --- a/eden/scm/lib/third-party/streampager/Cargo.toml +++ b/eden/scm/lib/third-party/streampager/Cargo.toml @@ -19,7 +19,7 @@ dirs = "6.0" enum_dispatch = "0.3.13" lru = { version = "0.16", default-features = false } memmap2 = "0.9.5" -notify = { version = "5", optional = true } +notify = { version = "6.1.1", optional = true } regex = "1.12.2" scopeguard = "1.2.0" serde = { version = "1.0.219", features = ["derive"] } diff --git a/eden/scm/sapling/cmdutil.py b/eden/scm/sapling/cmdutil.py index 1d6472ebbfebd..96b45dca45cb9 100644 --- a/eden/scm/sapling/cmdutil.py +++ b/eden/scm/sapling/cmdutil.py @@ -22,7 +22,7 @@ import tempfile import typing from enum import Enum -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import bindings from bindings import renderdag @@ -67,7 +67,7 @@ ) from .i18n import _, _x from .node import bin, hex, nullid, nullrev, short -from .utils import pathaclutil, subtreeutil +from .utils import iterutil, pathaclutil, subtreeutil if typing.TYPE_CHECKING: from .ui import ui @@ -3360,13 +3360,16 @@ def displaygraph( repo, dag, displayer, - getrenamed=None, - filematcher=None, + repogetrenamed=None, + repofilematcher=None, props=None, reserved=None, out=None, on_output=None, + graphnodeid_to_rev=None, ): + repogetrenamed = repogetrenamed or {} + repofilematcher = repofilematcher or {} props = props or {} formatnode = _graphnodeformatter(ui, displayer) if ui.plain("graph"): @@ -3399,12 +3402,24 @@ def displaygraph( renderer = renderer(minheight) if reserved: - for rev in reserved: - renderer.reserve(rev) + for graphnodeid in reserved: + renderer.reserve(graphnodeid) + + getrenamed = repogetrenamed.get(repo.root) + filematcher = repofilematcher.get(repo.root) + prev_repo = repo + xreponame = None show_abbreviated_ancestors = ShowAbbreviatedAncestorsWhen.load_from_config(repo.ui) - for rev, _type, ctx, parents in dag: - char = formatnode(repo, ctx) + for graphnodeid, _type, ctx, parents in dag: + curr_repo = ctx.repo() + if curr_repo.root != prev_repo.root: + getrenamed = repogetrenamed.get(curr_repo.root) + filematcher = repofilematcher.get(curr_repo.root) + xreponame = curr_repo.ui.config("remotefilelog", "reponame", "unknown") + prev_repo = curr_repo + + char = formatnode(curr_repo, ctx) copies = None if getrenamed and ctx.rev(): copies = [] @@ -3420,23 +3435,30 @@ def displaygraph( parents = [] elif show_abbreviated_ancestors is ShowAbbreviatedAncestorsWhen.NEVER: parents = [p for p in parents if p[0] != graphmod.MISSINGPARENT] - gprevs = [p[1] for p in parents if p[0] == graphmod.GRANDPARENT] + + gprevs = [ + graphnodeid_to_rev(p[1]) if graphnodeid_to_rev else p[1] + for p in parents + if p[0] == graphmod.GRANDPARENT + ] if all(isinstance(gp, bytes) for gp in gprevs): # parents are already nodes (when called from ext/commitcloud/commands.py) gpnodes = gprevs else: - gpnodes = repo.changelog.tonodes(gprevs) + gpnodes = curr_repo.changelog.tonodes(gprevs) revcache = {"copies": copies, "gpnodes": gpnodes} - width = renderer.width(rev, parents) + if xreponame: + revcache["xreponame"] = xreponame + width = renderer.width(graphnodeid, parents) displayer.show( ctx, revcache=revcache, matchfn=revmatchfn, _graphwidth=width, **props ) # The Rust graph renderer works with unicode. msg = "".join( s if isinstance(s, str) else s.decode(errors="replace") - for s in displayer.hunk.pop(rev) + for s in displayer.hunk.pop(ctx.rev()) ) - nextrow = renderer.nextrow(rev, parents, char, msg) + nextrow = renderer.nextrow(graphnodeid, parents, char, msg) if out is not None: out(nextrow) else: @@ -3481,11 +3503,33 @@ def load_from_config( ) -def graphlog(ui, repo, pats, opts): +def graphlog(ui, repo, pats: Tuple[str, ...], opts: Dict[str, Any]): # Parameters are identical to log command ones + repogetrenamed, repofilematcher, repoids = {}, {}, {} + revdag = _logdagwalker(repo, pats, opts, repogetrenamed, repofilematcher, repoids) + + ui.pager("log") + displayer = show_changeset(ui, repo, opts, buffered=True) + graphnodeid_to_rev = lambda graphnodeid: graphnodeid[1] + displaygraph( + ui, + repo, + revdag, + displayer, + repogetrenamed, + repofilematcher, + graphnodeid_to_rev=graphnodeid_to_rev, + ) + + +def _logdagwalker(repo, pats, opts, repogetrenamed, repofilematcher, repoids): revs, expr, filematcher = getgraphlogrevs(repo, pats, opts) template = opts.get("template") or "" - revdag = graphmod.dagwalker(repo, revs, template) + if repo.root not in repoids: + repoids[repo.root] = len(repoids) + rid = repoids[repo.root] + rev_to_graphnodeid = lambda rev: (rid, rev) + revdag = graphmod.dagwalker(repo, revs, template, idfunc=rev_to_graphnodeid) getrenamed = None if opts.get("copies"): @@ -3494,9 +3538,76 @@ def graphlog(ui, repo, pats, opts): endrev = scmutil.revrange(repo, opts.get("rev")).max() + 1 getrenamed = templatekw.getrenamedfn(repo, endrev=endrev) - ui.pager("log") - displayer = show_changeset(ui, repo, opts, buffered=True) - displaygraph(ui, repo, revdag, displayer, getrenamed, filematcher) + repogetrenamed[repo.root] = getrenamed + repofilematcher[repo.root] = filematcher + return _dagfollowxrepo( + repo, pats, opts, revdag, repogetrenamed, repofilematcher, repoids + ) + + +def _dagfollowxrepo(repo, pats, opts, revdag, repogetrenamed, repofilematcher, repoids): + for item, is_last in iterutil.mark_last(revdag): + # It currently doesn't handle multiple imports/merges. In those cases, the + # xrepo operation can appear in the middle of the dag. + if is_last: + xrepoinfo = xrepologinfo(repo, pats, opts, item[2]) + if not xrepoinfo: + yield item + break + + from_repo, from_commit, from_path = xrepoinfo + pats = [os.path.join(from_repo.root, from_path)] + opts = opts.copy() + opts["rev"] = [f"reverse(::{from_commit})"] + for from_item, is_first in iterutil.mark_first( + _logdagwalker( + from_repo, + pats, + opts, + repogetrenamed, + repofilematcher, + repoids, + ) + ): + if is_first: + # clear the in-repo missing parents and add cross-repo parents + item[3].clear() + item[3].append((graphmod.XREPOPARENT, from_item[0])) + yield item + yield from_item + else: + yield item + + +def xrepologinfo(curr_repo, curr_pats, curr_opts, lastctx): + if not curr_repo.ui.configbool("log", "follow-xrepo", True): + return None + + # XXX: currently supports only one path + if lastctx is None or len(curr_pats) != 1: + return None + match, pats = scmutil.matchandpats(curr_repo[None], curr_pats, curr_opts) + if len(match.files()) != 1: + return None + + curr_path = match.files()[0] + subtree_import = subtreeutil.find_subtree_import( + curr_repo, lastctx.node(), curr_path + ) + if not subtree_import: + return None + + from_url, from_commit, from_path = subtree_import + # XXX: currently only support import from git repo + git_url = git.maybegiturl(from_url) + if not git_url: + return None + + with curr_repo.ui.configoverride({("ui", "quiet"): True}): + from_repo = subtreeutil.get_or_clone_git_repo( + curr_repo.ui, git_url, from_commit + ) + return from_repo, from_commit, from_path def checkunsupportedgraphflags(pats, opts): @@ -3507,14 +3618,6 @@ def checkunsupportedgraphflags(pats, opts): ) -def graphrevs(repo, nodes, opts): - limit = loglimit(opts) - nodes.reverse() - if limit is not None: - nodes = nodes[:limit] - return graphmod.nodes(repo, nodes) - - def add(ui, repo, match, prefix, explicitonly, **opts): bad = [] diff --git a/eden/scm/sapling/commands/__init__.py b/eden/scm/sapling/commands/__init__.py index bef698a7a300b..e3761cd56f775 100644 --- a/eden/scm/sapling/commands/__init__.py +++ b/eden/scm/sapling/commands/__init__.py @@ -4080,7 +4080,7 @@ def log(ui, repo, *pats, **opts): if not lastctx: break - xrepoinfo = _getxrepoinfo(curr_repo, curr_pats, curr_opts, lastctx) + xrepoinfo = cmdutil.xrepologinfo(curr_repo, curr_pats, curr_opts, lastctx) if not xrepoinfo: break @@ -4094,34 +4094,6 @@ def log(ui, repo, *pats, **opts): curr_opts["rev"] = [f"reverse(::{from_commit})"] -def _getxrepoinfo(curr_repo, curr_pats, curr_opts, lastctx): - # XXX: currently supports only one path - if lastctx is None or len(curr_pats) != 1: - return None - match, pats = scmutil.matchandpats(curr_repo[None], curr_pats, curr_opts) - if len(match.files()) != 1: - return None - - curr_path = match.files()[0] - subtree_import = subtreeutil.find_subtree_import( - curr_repo, lastctx.node(), curr_path - ) - if not subtree_import: - return None - - from_url, from_commit, from_path = subtree_import - # XXX: currently only support import from git repo - git_url = git.maybegiturl(from_url) - if not git_url: - return None - - with curr_repo.ui.configoverride({("ui", "quiet"): True}): - from_repo = subtreeutil.get_or_clone_git_repo( - curr_repo.ui, git_url, from_commit - ) - return from_repo, from_commit, from_path - - def _dolog(ui, repo, pats, opts, count, xreponame): revs, expr, filematcher = cmdutil.getlogrevs(repo, pats, opts) hunksfilter = None diff --git a/eden/scm/sapling/commands/debug.py b/eden/scm/sapling/commands/debug.py index a6cd70bd37e12..9218170b0ca38 100644 --- a/eden/scm/sapling/commands/debug.py +++ b/eden/scm/sapling/commands/debug.py @@ -351,7 +351,7 @@ def debugbisect(repo, ctx, templ, **args): cmdutil.graphlog( ui, repo, - [], + (), { "rev": [hex(n) for n in node_to_infos], "template": "{desc|firstline}: {debugbisect}", @@ -2407,19 +2407,23 @@ def debugpreviewbindag(ui, repo, path): data = util.readfile(path) revs, parentrevs = dagparser.parsebindag(data) - def gendag(revs, parentrevs): + def gendag(repo, revs, parentrevs): for rev in reversed(revs): - yield (rev, "C", dummyctx(rev), [("P", p) for p in parentrevs(rev)]) + yield (rev, "C", dummyctx(repo, rev), [("P", p) for p in parentrevs(rev)]) class dummyctx: """A dummy changeset object""" - def __init__(self, id): + def __init__(self, repo, id): self.id = id + self._repo = repo def rev(self): return self.id + def repo(self): + return self._repo + # required methods by displayer def node(self): @@ -2430,7 +2434,7 @@ def obsolete(self): opts = {"template": "{rev}", "graph": True} displayer = cmdutil.show_changeset(ui, repo, opts, buffered=True) - cmdutil.displaygraph(ui, repo, gendag(revs, parentrevs), displayer) + cmdutil.displaygraph(ui, repo, gendag(repo, revs, parentrevs), displayer) @command( diff --git a/eden/scm/sapling/ext/undo.py b/eden/scm/sapling/ext/undo.py index c09532cc3d4a9..219997f27a2a1 100644 --- a/eden/scm/sapling/ext/undo.py +++ b/eden/scm/sapling/ext/undo.py @@ -1418,7 +1418,7 @@ def _preview(ui, repo, reverseindex): opts["rev"] = [revstring] try: with ui.configoverride(overrides): - cmdutil.graphlog(ui, repo, None, opts) + cmdutil.graphlog(ui, repo, (), opts) # informative output nodedict = _readindex(repo, reverseindex) time = _readnode(repo, "date.i", nodedict["date"]) diff --git a/eden/scm/sapling/graphmod.py b/eden/scm/sapling/graphmod.py index 503ff7e64cf18..4a93653550936 100644 --- a/eden/scm/sapling/graphmod.py +++ b/eden/scm/sapling/graphmod.py @@ -28,6 +28,7 @@ CHANGESET = "C" PARENT = "P" GRANDPARENT = "G" +XREPOPARENT = "X" MISSINGPARENT = "M" # Style of line to draw. None signals a line that ends and is removed at this # point. A number prefix means only the last N characters of the current block @@ -36,7 +37,7 @@ EDGES = {PARENT: "|", GRANDPARENT: ":", MISSINGPARENT: None} -def dagwalker(repo, revs, template): +def dagwalker(repo, revs, template, idfunc=None): """cset DAG generator yielding (id, CHANGESET, ctx, [parentinfo]) tuples This generator function walks through revisions (which should be ordered @@ -47,6 +48,7 @@ def dagwalker(repo, revs, template): are arbitrary integers which identify a node in the context of the graph returned. + The idfunc is a function that takes a rev number and returns an id for it. """ if not revs: return @@ -58,6 +60,9 @@ def dagwalker(repo, revs, template): if simplifygrandparents: rootnodes = cl.tonodes(revs) + if idfunc is None: + idfunc = lambda rev: rev + gpcache = {} ctxstream = revs.prefetchbytemplate(repo, template).iterctx() for ctx in ctxstream: @@ -68,7 +73,7 @@ def dagwalker(repo, revs, template): mpars = [ p.rev() for p in ctx.parents() if p.rev() != nullrev and p.rev() not in pset ] - parents = [(PARENT, p) for p in sorted(pset)] + parents = [(PARENT, idfunc(p)) for p in sorted(pset)] for mpar in mpars: gp = gpcache.get(mpar) @@ -91,23 +96,10 @@ def dagwalker(repo, revs, template): set(dagop.reachableroots(repo, revs, [mpar])) ) if not gp: - parents.append((MISSINGPARENT, mpar)) + parents.append((MISSINGPARENT, idfunc(mpar))) pset.add(mpar) else: - parents.extend((GRANDPARENT, g) for g in gp if g not in pset) + parents.extend((GRANDPARENT, idfunc(g)) for g in gp if g not in pset) pset.update(gp) - yield (ctx.rev(), CHANGESET, ctx, parents) - - -def nodes(repo, nodes): - """cset DAG generator yielding (id, CHANGESET, ctx, [parentids]) tuples - - This generator function walks the given nodes. It only returns parents - that are in nodes, too. - """ - include = set(nodes) - for node in nodes: - ctx = repo[node] - parents = set((PARENT, p.rev()) for p in ctx.parents() if p.node() in include) - yield (ctx.rev(), CHANGESET, ctx, sorted(parents)) + yield (idfunc(ctx.rev()), CHANGESET, ctx, parents) diff --git a/eden/scm/sapling/utils/iterutil.py b/eden/scm/sapling/utils/iterutil.py new file mode 100644 index 0000000000000..50ca1f4840be2 --- /dev/null +++ b/eden/scm/sapling/utils/iterutil.py @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2. + + +def mark_first(iterable): + """Yield (item, is_first) tuples from an iterable. + + >>> iterable = [1, 2, 3, 4] + >>> for i in mark_first(iterable): + ... print(i) + ... + (1, True) + (2, False) + (3, False) + (4, False) + """ + it = iter(iterable) + first = True + for a in it: + yield a, first + first = False + + +def mark_last(iterable): + """Yield (item, is_last) tuples from an iterable. + + >>> iterable = [1, 2, 3, 4] + >>> for i in mark_last(iterable): + ... print(i) + ... + (1, False) + (2, False) + (3, False) + (4, True) + """ + it = iter(iterable) + for a in it: + for b in it: + yield a, False + a = b + yield a, True diff --git a/eden/scm/saplingnative/bindings/modules/pyrenderdag/src/lib.rs b/eden/scm/saplingnative/bindings/modules/pyrenderdag/src/lib.rs index f2b4363589432..8c119007d37ad 100644 --- a/eden/scm/saplingnative/bindings/modules/pyrenderdag/src/lib.rs +++ b/eden/scm/saplingnative/bindings/modules/pyrenderdag/src/lib.rs @@ -53,8 +53,16 @@ impl<'a> FromPyObject<'a> for PyNode { } else if let Ok(rev) = obj.extract::(py) { let slice: [u8; 8] = rev.to_be_bytes(); Ok(PyNode(Bytes::copy_from_slice(&slice))) + } else if let Ok(tuple) = obj.extract::<(u8, i64)>(py) { + let mut bytes = Vec::with_capacity(9); + bytes.extend_from_slice(&tuple.0.to_be_bytes()); + bytes.extend_from_slice(&tuple.1.to_be_bytes()); + Ok(PyNode(Bytes::from(bytes))) } else { - Err(PyErr::new::(py, "expect bytes or int")) + Err(PyErr::new::( + py, + "expect bytes, int or (u8, i64) tuple", + )) } } } @@ -65,6 +73,7 @@ fn convert_parents(py: Python, parents: Vec<(String, PyNode)>) -> PyResult Ok(Ancestor::Parent(parent.into())), "G" => Ok(Ancestor::Ancestor(parent.into())), + "X" => Ok(Ancestor::Ancestor(parent.into())), "M" => Ok(Ancestor::Anonymous), _ => Err(PyErr::new::( py, diff --git a/eden/scm/tests/test-doctest.py b/eden/scm/tests/test-doctest.py index 34bbd8324df71..9b5f622d527ee 100644 --- a/eden/scm/tests/test-doctest.py +++ b/eden/scm/tests/test-doctest.py @@ -78,6 +78,7 @@ def testmod(name, optionflags=0, testtarget=None): testmod("sapling.url") testmod("sapling.util") testmod("sapling.util", testtarget="platform") +testmod("sapling.utils.iterutil") testmod("sapling.ext.commitcloud.sync") testmod("sapling.ext.fbcodereview") testmod("sapling.ext.remotenames") diff --git a/eden/scm/tests/test-subtree-import-log.t b/eden/scm/tests/test-subtree-import-log.t index f301bc24625e1..e44896b96a5bd 100644 --- a/eden/scm/tests/test-subtree-import-log.t +++ b/eden/scm/tests/test-subtree-import-log.t @@ -132,3 +132,72 @@ Test xreponame keyword gitrepo gitrepo + +Test log with --graph + $ hg log bar/alpha --graph + @ commit: * (glob) + │ user: test + │ date: Thu Jan 01 00:00:00 1970 +0000 + │ summary: update bar/alpha + │ + o commit: * (glob) + ╷ user: test + ╷ date: Thu Jan 01 00:00:00 1970 +0000 + ╷ summary: import gitrepo to bar + ╷ + @ commit: 6a5b13188f04~ + │ user: test + │ date: Mon Jan 01 00:00:10 2007 +0000 + │ summary: update alpha\nhttps://phabricator.test.com/D1234567 + │ + o commit: b6c31add3e60~ + user: test + date: Mon Jan 01 00:00:10 2007 +0000 + summary: alpha + +Test commit color with --graph + + $ hg log bar/alpha --graph --color=debug + @ [log.changeset changeset.draft|commit: *] (glob) + │ [log.user|user: test] + │ [log.date|date: Thu Jan 01 00:00:00 1970 +0000] + │ [log.summary|summary: update bar/alpha] + │ + o [log.changeset changeset.draft|commit: *] (glob) + ╷ [log.user|user: test] + ╷ [log.date|date: Thu Jan 01 00:00:00 1970 +0000] + ╷ [log.summary|summary: import gitrepo to bar] + ╷ + @ [log.changeset changeset.public changeset.xrepo|commit: 6a5b13188f04~] + │ [log.user|user: test ] + │ [log.date|date: Mon Jan 01 00:00:10 2007 +0000] + │ [log.summary|summary: update alpha\nhttps://phabricator.test.com/D1234567] + │ + o [log.changeset changeset.public changeset.xrepo|commit: b6c31add3e60~] + [log.user|user: test ] + [log.date|date: Mon Jan 01 00:00:10 2007 +0000] + [log.summary|summary: alpha] + +Test log.follow-xrepo config + + $ hg log bar/alpha --config log.follow-xrepo=False + commit: * (glob) + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + summary: update bar/alpha + + commit: * (glob) + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + summary: import gitrepo to bar + + $ hg log bar/alpha --graph --config log.follow-xrepo=False + @ commit: * (glob) + │ user: test + │ date: Thu Jan 01 00:00:00 1970 +0000 + │ summary: update bar/alpha + │ + o commit: * (glob) + │ user: test + ~ date: Thu Jan 01 00:00:00 1970 +0000 + summary: import gitrepo to bar