Merge branch 'feature/add_component_manager_python_dependency' into 'master'

Build System: Add idf-component-manager python dependency

Closes PACMAN-172 and DOC-1623

See merge request espressif/esp-idf!13432

.gitignore

@@ -88,3 +88,6 @@ build
 
 # lock files for examples and components
 dependencies.lock
+
+# managed_components for examples
+managed_components

docs/en/api-guides/tools/idf-component-manager.rst

@@ -0,0 +1,72 @@
+*********************
+IDF Component Manager
+*********************
+
+The IDF Component manager is a tool that downloads dependencies for any ESP-IDF CMake project. The download happens automatically during a run of CMake. It can source components either from `the component registry <https://components.espressif.com>`_ or from a git repository.
+
+A list of components can be found on `<https://components.espressif.com/>`_
+
+Activating the Component Manager
+================================
+
+If CMake is started using ``idf.py`` or `ESP-IDF VSCode Extension <https://marketplace.visualstudio.com/items?itemName=espressif.esp-idf-extension>`_ then the component manager will be activated by default.
+
+If CMake is used directly or with some CMake-based IDE like CLion, it's necessary to set the ``IDF_COMPONENT_MANAGER`` environment variable to ``1`` to enable the component manager integration with the build system.
+
+Using with a project
+====================
+
+Dependencies for each component in the project are defined in a separate manifest file named ``idf_component.yml`` placed in the root of the component. The manifest file template can be created for a component by running ``idf.py create-manifest --component=my_component``. When a new manifest is added to one of the components in the project it's necessary to reconfigure it manually by running ``idf.py reconfigure``. Then build will track changes in ``idf_component.yml`` manifests and automatically triggers CMake when necessary.
+
+There is an example application: example:`build_system/cmake/component_manager` that uses components installed by the component manager.
+
+It's not necessary to have a manifest for components that don't need any managed dependencies.
+
+When CMake configures the project (e.g. ``idf.py reconfigure``) component manager does a few things:
+
+- Processes ``idf_component.yml`` manifests for every component in the project and recursively solves dependencies
+- Creates a ``dependencies.lock`` file in the root of the project with a full list of dependencies
+- Downloads all dependencies to the ``managed_components`` directory
+
+The lock-file ``dependencies.lock`` and content of ``managed_components`` directory is not supposed to be modified by a user. When the component manager runs it always make sure they are up to date. If these files were accidentally modified it's possible to re-run the component manager by triggering CMake with ``idf.py reconfigure``
+
+Defining dependencies in the manifest
+=====================================
+
+.. code-block:: yaml
+
+    dependencies:
+      # Required IDF version
+      idf: ">=4.1"
+      # Defining a dependency from the registry:
+      # https://components.espressif.com/component/example/cmp
+      example/cmp: ">=1.0.0"
+
+      # # Other ways to define dependencies
+      #
+      # # For components maintained by Espressif only name can be used.
+      # # Same as `espressif/cmp`
+      # component: "~1.0.0"
+      #
+      # # Or in a longer form with extra parameters
+      # component2:
+      #   version: ">=2.0.0"
+      #
+      #   # For transient dependencies `public` flag can be set.
+      #   # `public` flag doesn't affect the `main` component.
+      #   # All dependencies of `main` are public by default.
+      #   public: true
+      #
+      #   # For components hosted on non-default registry:
+      #   service_url: "https://componentregistry.company.com"
+      #
+      # # For components in git repository:
+      # test_component:
+      #   path: test_component
+      #   git: ssh://git@gitlab.com/user/components.git
+      #
+      # # For test projects during component development
+      # # components can be used from a local directory
+      # # with relative or absolute path
+      # some_local_component:
+      #   path: ../../projects/component

docs/en/api-guides/tools/index.rst

@@ -8,3 +8,4 @@ Tools
    IDF Monitor <idf-monitor>
    IDF Docker image <idf-docker-image>
    IDF Windows Installer <idf-windows-installer>
+   IDF Component Manager <idf-component-manager>

docs/zh_CN/api-guides/tools/idf-component-manager.rst

@@ -0,0 +1 @@
+.. include:: ../../../en/api-guides/tools/idf-component-manager.rst

docs/zh_CN/api-guides/tools/index.rst

@@ -8,3 +8,4 @@
    IDF 监视器 <idf-monitor>
    IDF Docker image <idf-docker-image>
    IDF Windows Installer <idf-windows-installer>
+   IDF Component Manager <idf-component-manager>

examples/build_system/cmake/component_manager/CMakeLists.txt

@@ -0,0 +1,8 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(component_manager)

examples/build_system/cmake/component_manager/README.md

@@ -0,0 +1,55 @@
+# Using the component manager for downloading dependencies
+
+This example demonstrates how to use [IDF Component Manager](https://pypi.org/project/idf-component-manager/) for downloading dependencies from [the component registry](https://components.espressif.com). More details and use cases of IDF Component Manager can be found in the programming guide under `API Guides` -> `Tools` -> `IDF Component Manager`.
+
+## How to use the example
+### Hardware Required
+
+This example is designed to work with any commonly available development kit.
+
+### Build and Flash
+
+Run `idf.py reconfigure` to configure this project. During CMake execution the component manager will process data from the manifest file `./main/idf_component.yml` where 2 dependencies are defined:
+
+- `idf: ">=4.1"` - Specifies required version of ESP-IDF.
+- `example/cmp: ">=3.3.3"` - Defines dependency on [example/cmp](https://components.espressif.com/component/example/cmp) component that is used by the main component.
+
+CMake Output:
+```
+...
+Solving dependencies requirements
+Updating lock file at /home/user/esp-idf/examples/build_system/cmake/component_manager/dependencies.lock
+Processing 2 dependencies:
+[1/2] example/cmp
+[2/2] idf
+...
+```
+
+Content of the `./managed_components` directory after successful build:
+```
+> find  ./managed_components
+./managed_components
+./managed_components/example__cmp
+./managed_components/example__cmp/include
+./managed_components/example__cmp/include/cmp.h
+./managed_components/example__cmp/LICENSE
+./managed_components/example__cmp/README.md
+./managed_components/example__cmp/CMakeLists.txt
+./managed_components/example__cmp/changelog.md
+./managed_components/example__cmp/cmp.c
+./managed_components/example__cmp/idf_component.yml
+```
+
+Flash the project and run the serial monitor to view the output:
+
+```
+idf.py -p PORT flash monitor
+```
+
+### Example Output
+
+The example outputs a line from the `cmp_hello` function from the component downloaded by the component manager.
+
+```
+Hello from example component!
+```

examples/build_system/cmake/component_manager/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "component_manager.c"
+                    INCLUDE_DIRS ".")

examples/build_system/cmake/component_manager/main/component_manager.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+#include "cmp.h"
+
+void app_main(void)
+{
+    cmp_hello();
+}

examples/build_system/cmake/component_manager/main/idf_component.yml

@@ -0,0 +1,36 @@
+dependencies:
+  # Required IDF version
+  idf: ">=4.1"
+
+  # Defining a dependency from the registry:
+  # https://components.espressif.com/component/example/cmp
+  example/cmp: ">=3.3.3"
+
+  # # Other ways to define dependencies
+  #
+  # # For components maintained by Espressif only name can be used.
+  # # Same as `espressif/cmp`
+  # component: "~1.0.0"
+  #
+  # # Or in a longer form with extra parameters
+  # component2:
+  #   version: ">=2.0.0"
+  #
+  #   # For transient dependencies `public` flag can be set.
+  #   # `public` flag doesn't have an effect for the `main` component.
+  #   # All dependencies of `main` are public by default.
+  #   public: true
+  #
+  #   # For components hosted on non-default registry:
+  #   service_url: "https://componentregistry.company.com"
+  #
+  # # For components in git repository:
+  # test_component:
+  #   path: test_component
+  #   git: ssh://git@gitlab.com/user/components.git
+  #
+  # # For test projects during component development
+  # # components can be used from a local directory
+  # # with relative or absolute path
+  # some_local_component:
+  #   path: ../../projects/component

requirements.txt

@@ -11,6 +11,7 @@ future>=0.15.2
 cryptography>=2.1.4
 pyparsing>=2.0.3,<2.4.0
 pyelftools>=0.22
+idf-component-manager>=0.2.99-beta
 
 gdbgui==0.13.2.0
 # 0.13.2.1 supports Python 3.6+ only

tools/ci/test_build_system_cmake.sh

@@ -717,14 +717,9 @@ endmenu\n" >> ${IDF_PATH}/Kconfig
 
     print_status "Compiles with dependencies delivered by component manager"
     clean_build_dir
-    # Make sure that component manager is not installed
-    pip uninstall -y idf_component_manager
     printf "\n#include \"test_component.h\"\n" >> main/main.c
     printf "dependencies:\n  test_component:\n    path: test_component\n    git: ${COMPONENT_MANAGER_TEST_REPO}\n" >> main/idf_component.yml
-    ! idf.py build || failure "Build should fail if dependencies are not installed"
-    pip install ${COMPONENT_MANAGER_PACKAGE} --upgrade || failure "Failed to install component manager"
     idf.py reconfigure build || failure "Build didn't succeed with required components installed by package manager"
-    pip uninstall -y idf_component_manager
     rm main/idf_component.yml
     git checkout main/main.c
 

tools/cmake/component.cmake

@@ -434,9 +434,8 @@ function(idf_component_register)
     __component_check_target()
     __component_add_sources(sources)
 
-    # Add component manifest and lock files to list of dependencies
+    # Add component manifest to the list of dependencies
     set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/idf_component.yml")
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/dependencies.lock")
 
     # Create the final target for the component. This target is the target that is
     # visible outside the build system.