diff --git a/.github/workflows/ports_esp32.yml b/.github/workflows/ports_esp32.yml
index 9403aa656b..117ae897aa 100644
--- a/.github/workflows/ports_esp32.yml
+++ b/.github/workflows/ports_esp32.yml
@@ -22,7 +22,25 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v4
-    - name: Install packages
-      run: source tools/ci.sh && ci_esp32_idf50_setup
+
+    - id: idf_ver
+      name: Read the ESP-IDF version
+      run: source tools/ci.sh && echo "IDF_VER=$IDF_VER" | tee "$GITHUB_OUTPUT"
+
+    - name: Cached ESP-IDF install
+      id: cache_esp_idf
+      uses: actions/cache@v3
+      with:
+        path: |
+          ./esp-idf/
+          ~/.espressif/
+          !~/.espressif/dist/
+          ~/.cache/pip/
+        key: esp-idf-${{ steps.idf_ver.outputs.IDF_VER }}
+
+    - name: Install ESP-IDF packages
+      if: steps.cache_esp_idf.outputs.cache-hit != 'true'
+      run: source tools/ci.sh && ci_esp32_idf_setup
+
     - name: Build
       run: source tools/ci.sh && ci_esp32_build
diff --git a/tools/ci.sh b/tools/ci.sh
index 8dcbbf7765..34260334d8 100755
--- a/tools/ci.sh
+++ b/tools/ci.sh
@@ -118,10 +118,15 @@ function ci_cc3200_build {
 ########################################################################################
 # ports/esp32
 
-function ci_esp32_idf50_setup {
+# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch)
+IDF_VER=v5.0.2
+
+function ci_esp32_idf_setup {
     pip3 install pyelftools
-    git clone https://github.com/espressif/esp-idf.git
-    git -C esp-idf checkout v5.0.2
+    git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git
+    # doing a treeless clone isn't quite as good as --shallow-submodules, but it
+    # is smaller than full clones and works when the submodule commit isn't a head.
+    git -C esp-idf submodule update --init --recursive --filter=tree:0
     ./esp-idf/install.sh
 }