test: migrate linux target host test jobs into pytest-embedded

.gitlab/ci/host-test.yml

@@ -20,13 +20,6 @@ test_nvs_on_host:
     - cd components/nvs_flash/test_nvs_host
     - make test
 
-test_nvs_on_host_cmake:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_host_test
-    - idf.py build
-    - build/nvs_host_test.elf
-
 test_nvs_coverage:
   extends:
     - .host_test_template
@@ -340,13 +333,6 @@ test_split_path_by_spaces:
     - cd ${IDF_PATH}/tools
     - python -m unittest split_paths_by_spaces.py
 
-test_nvs_page:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_page_test
-    - idf.py build
-    - build/test_nvs_page_host.elf
-
 test_mqtt_on_host:
   extends: .host_test_template
   script:
@@ -354,60 +340,6 @@ test_mqtt_on_host:
     - idf.py build
     - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
 
-test_log:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/log/host_test/log_test
-    - idf.py build
-    - build/test_log_host.elf
-
-test_esp_event:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_event/host_test/esp_event_unit_test
-    - idf.py build
-    - build/test_esp_event_host.elf
-
-test_hello_world_linux_compatible_example:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/tools/test_apps/linux_compatible/hello_world_linux_compatible
-    - idf.py --preview set-target linux
-    - idf.py build
-    - timeout 15 build/hello_world.elf | tee test.txt
-    - grep "Hello world!" test.txt
-
-test_esp_system:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_system/host_test/test_esp_system/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_esp_system.elf | tee log.txt || true
-    - grep "6 Tests 0 Failures 0 Ignored" log.txt
-
-test_heap_linux:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/heap/host_test/host_test_linux/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_heap.elf | tee log.txt || true
-    - grep "4 Tests 0 Failures 0 Ignored" log.txt
-
-test_esp_hw_support_linux:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_hw_support/host_test/host_test_linux/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_hw_support_linux.elf | tee log.txt || true
-    - grep "2 Tests 0 Failures 0 Ignored" log.txt
-
-test_esp_timer_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/esp_timer
-    - idf.py build
-    - build/test_esp_timer_cxx_host.elf
-
 test_eh_frame_parser:
   extends: .host_test_template
   script:
@@ -415,57 +347,6 @@ test_eh_frame_parser:
     - make
     - ./eh_frame_test
 
-test_rom_on_linux_works:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_rom/host_test/rom_test
-    - idf.py build
-    - build/test_rom_host.elf
-
-test_cxx_gpio:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/gpio
-    - idf.py build
-    - build/test_gpio_cxx_host.elf
-
-test_i2c_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/i2c
-    - idf.py build
-    - build/test_i2c_cxx_host.elf
-
-test_spi_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/spi
-    - idf.py build
-    - build/test_spi_cxx_host.elf
-
-test_system_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/system
-    - idf.py build
-    - build/test_system_cxx_host.elf
-
-test_partition_api_host:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
-    - idf.py build
-    - timeout 5 ./build/partition_api_test.elf | tee test.txt
-    - grep " 0 Failures" test.txt
-
-test_spiffs_host:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/spiffs/host_test
-    - idf.py build
-    - timeout 5 ./build/host_test_spiffs.elf | tee test.txt
-    - grep " 0 Failures" test.txt
-
 test_gen_soc_caps_kconfig:
   extends: .host_test_template
   script:
@@ -497,3 +378,26 @@ test_pytest_qemu:
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --known-failure-cases-file known_failure_cases/known_failure_cases.txt
+
+test_pytest_linux:
+  extends:
+    - .host_test_template
+  artifacts:
+    when: always
+    paths:
+      - XUNIT_RESULT.xml
+      - pytest_embedded_log/
+    reports:
+      junit: XUNIT_RESULT.xml
+    expire_in: 1 week
+  script:
+    # TODO: fix the warnings in build flags and ignore-warning-str: IDF-6637
+    # Record the warnings regexes in file tools/ci/ignore_build_warnings_linux.txt to avoid using parentheses and
+    # doublequotes with bash.
+    # Please remove that file while fixing all the warnings.
+    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
+      --target linux
+      --pytest-apps
+      -m host_test
+      --ignore-warning-file tools/ci/ignore_build_warnings_linux.txt
+    - run_cmd pytest --target linux -m host_test --junitxml=XUNIT_RESULT.xml

components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_event_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/esp_hw_support/host_test/host_test_linux/pytest_esp_hw_support_linux.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_hw_support_linux(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests.')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=5)

components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_partition_linux(dut: Dut) -> None:
+    dut.expect_unity_test_output(timeout=5)

components/esp_rom/host_test/rom_test/pytest_esp_rom_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_rom_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/esp_system/host_test/test_esp_system/pytest_esp_system_linux.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_system_linux(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests.')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=10)

components/heap/host_test/host_test_linux/pytest_heap_linux.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_heap_linux(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests.')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=10)

components/log/host_test/log_test/pytest_log_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_log_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_nvs_host_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_nvs_page_linux(dut: Dut) -> None:
+    dut.expect_unity_test_output(timeout=10)

components/spiffs/host_test/pytest_spiffs_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_spiffs_linux(dut: Dut) -> None:
+    dut.expect_unity_test_output(timeout=5)

examples/cxx/experimental/experimental_cpp_component/host_test/esp_timer/pytest_esp_timer_cxx.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_timer_cxx(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

examples/cxx/experimental/experimental_cpp_component/host_test/gpio/pytest_gpio_cxx.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_gpio_cxx(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

examples/cxx/experimental/experimental_cpp_component/host_test/i2c/pytest_i2c_cxx.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_i2c_cxx(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

examples/cxx/experimental/experimental_cpp_component/host_test/spi/pytest_spi_cxx.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_spi_cxx(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

examples/cxx/experimental/experimental_cpp_component/host_test/system/pytest_system_cxx.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_system_cxx(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

tools/ci/check_copyright_config.yaml

@@ -53,6 +53,7 @@ examples_and_unit_tests:
    - 'components/**/test/**'
    - 'components/**/test_apps/**'
    - 'tools/test_apps/**'
+   - '**/pytest_*.py'
   allowed_licenses:
   - Apache-2.0
   - Unlicense
@@ -145,6 +146,7 @@ xtensa:
   allowed_licenses:
     - Apache-2.0    #Files added to the xtensa component by us
     - MIT           #Cadence sources
+
 tinyusb:
   include:
     - 'examples/peripherals/usb/device/tusb_midi/'

tools/ci/exclude_check_tools_files.txt

@@ -25,6 +25,7 @@ tools/ci/fix_empty_prototypes.sh
 tools/ci/get-full-sources.sh
 tools/ci/idf_ci_utils.py
 tools/ci/ignore_build_warnings.txt
+tools/ci/ignore_build_warnings_linux.txt
 tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/mypy_ignore_list.txt

tools/ci/ignore_build_warnings_linux.txt

@@ -0,0 +1,6 @@
+the hex symbol ESP_MAIN_TASK_AFFINITY \(defined at .+\) has a non-hex default FREERTOS_NO_AFFINITY \(undefined\)
+"CONFIG_ESP_SYSTEM_EVENT_QUEUE_SIZE" redefined
+No function prototypes found!
+-Wdeprecated-declarations
+-Wno-enum-conversion
+-Wunused-result

tools/test_apps/.build-test-rules.yml

@@ -20,10 +20,6 @@ tools/test_apps/build_system/rsource_test:
 tools/test_apps/linux_compatible/hello_world_linux_compatible:
   enable:
     - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux"
-  disable_test:
-    - if: IDF_TARGET not in ["esp32", "esp32c3"]
-      temporary: true
-      reason: pytest doesn't support linux target yet, hence, it's tested independenly in the host_tests stage
 
 tools/test_apps/peripherals/usb:
   enable:

tools/test_apps/linux_compatible/hello_world_linux_compatible/pytest_hello_world_linux_compatible.py

@@ -1,15 +1,17 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: CC0-1.0
 
 import pytest
 from pytest_embedded_idf.dut import IdfDut
 
-# Note that support for Linux target console applications hasn't been implemented for pytest-embedded yet
-# (https://github.com/espressif/pytest-embedded/issues/106)
 
-
-@pytest.mark.esp32
-@pytest.mark.esp32c3
+@pytest.mark.supported_targets
 @pytest.mark.generic
 def test_hello_world_linux_compatible(dut: IdfDut) -> None:
     dut.expect('Hello world!')
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_hello_world_linux(dut: IdfDut) -> None:
+    dut.expect('Hello world!')