diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh
index 3770189b447..5b46e62067f 100755
--- a/.ci/docker/build.sh
+++ b/.ci/docker/build.sh
@@ -23,9 +23,14 @@ MINICONDA_VERSION=23.10.0-1
 BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
 
 case "${IMAGE_NAME}" in
-  executorch-ubuntu-22.04-gcc9)
+  executorch-ubuntu-22.04-gcc11)
+    LINTRUNNER=""
+    GCC_VERSION=11
+    ;;
+  executorch-ubuntu-22.04-gcc9-nopytorch)
     LINTRUNNER=""
     GCC_VERSION=9
+    SKIP_PYTORCH=yes
     ;;
   executorch-ubuntu-22.04-clang12)
     LINTRUNNER=""
@@ -95,6 +100,7 @@ docker build \
   --build-arg "QNN_SDK=${QNN_SDK:-}" \
   --build-arg "MEDIATEK_SDK=${MEDIATEK_SDK:-}" \
   --build-arg "ANDROID_NDK_VERSION=${ANDROID_NDK_VERSION:-}" \
+  --build-arg "SKIP_PYTORCH=${SKIP_PYTORCH:-}" \
   -f "${OS}"/Dockerfile \
   "$@" \
   .
diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile
index 9c57e5ee951..b7478df5489 100644
--- a/.ci/docker/ubuntu/Dockerfile
+++ b/.ci/docker/ubuntu/Dockerfile
@@ -64,9 +64,10 @@ ENV SCCACHE_S3_KEY_PREFIX executorch
 ENV SCCACHE_REGION us-east-1
 
 ARG TORCH_VERSION
+ARG SKIP_PYTORCH
 COPY ./common/install_pytorch.sh install_pytorch.sh
 COPY ./common/utils.sh utils.sh
-RUN bash ./install_pytorch.sh && rm install_pytorch.sh utils.sh
+RUN if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh utils.sh
 
 ARG LINTRUNNER
 # Install lintrunner if needed
diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml
index bfe0d5d752d..e3b72a6bcd6 100644
--- a/.github/workflows/docker-builds.yml
+++ b/.github/workflows/docker-builds.yml
@@ -33,7 +33,8 @@ jobs:
       matrix:
         runner: [linux.4xlarge]
         docker-image-name: [
-          executorch-ubuntu-22.04-gcc9,
+          executorch-ubuntu-22.04-gcc11,
+          executorch-ubuntu-22.04-gcc9-nopytorch,
           executorch-ubuntu-22.04-clang12,
           executorch-ubuntu-22.04-linter,
           executorch-ubuntu-22.04-arm-sdk,
@@ -70,7 +71,7 @@ jobs:
 
       - name: Build docker image
         id: build-docker-image
-        timeout-minutes: 45
+        timeout-minutes: 145
         uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
         with:
           docker-image-name: ci-image:${{ matrix.docker-image-name }}
diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
index b4566bdb35a..71161bc39f7 100644
--- a/.github/workflows/pull.yml
+++ b/.github/workflows/pull.yml
@@ -51,7 +51,7 @@ jobs:
       fail-fast: false
     with:
       runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-gcc9
+      docker-image: ci-image:executorch-ubuntu-22.04-gcc11
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
@@ -391,7 +391,7 @@ jobs:
       fail-fast: false
     with:
       runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-gcc9
+      docker-image: ci-image:executorch-ubuntu-22.04-gcc9-nopytorch
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
@@ -400,15 +400,16 @@ jobs:
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
         conda activate "${CONDA_ENV}"
 
-        ./install_requirements.sh --use-pt-pinned-commit
+        ./install_requirements.sh
         # build module for executorch.extension.pybindings.portable_lib
         bash test/build_size_test.sh
         strip cmake-out/test/size_test
         output=$(ls -la cmake-out/test/size_test)
         arr=($output)
         size=${arr[4]}
-        # threshold=48120 on devserver with gcc11.4
+        # threshold=48120 on devserver with gcc9
         # todo(lfq): update once binary size is below 50kb.
+        # Note: using gcc9-nopytorch container with pinned nightly PyTorch
         threshold="63776"
         if [[ "$size" -le "$threshold" ]]; then
           echo "Success $size <= $threshold"
@@ -793,7 +794,7 @@ jobs:
       fail-fast: false
     with:
       runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-gcc9
+      docker-image: ci-image:executorch-ubuntu-22.04-gcc11
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90