Merge pull request #1248 from minrk/memoize-getters

avoid duplicate log statements by memoizing getters

repo2docker/buildpacks/base.py

@@ -7,6 +7,7 @@ import string
 import sys
 import tarfile
 import textwrap
+from functools import lru_cache
 
 import escapism
 import jinja2
@@ -231,6 +232,7 @@ class BuildPack:
             )
         self.platform = ""
 
+    @lru_cache()
     def get_packages(self):
         """
         List of packages that are installed in this BuildPack.
@@ -240,6 +242,7 @@ class BuildPack:
         """
         return set()
 
+    @lru_cache()
     def get_base_packages(self):
         """
         Base set of apt packages that are installed for all images.
@@ -256,6 +259,7 @@ class BuildPack:
             "unzip",
         }
 
+    @lru_cache()
     def get_build_env(self):
         """
         Ordered list of environment variables to be set for this image.
@@ -271,6 +275,7 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_env(self):
         """
         Ordered list of environment variables to be set for this image.
@@ -285,6 +290,7 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_path(self):
         """
         Ordered list of file system paths to look for executables in.
@@ -294,12 +300,14 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_labels(self):
         """
         Docker labels to set on the built image.
         """
         return self.labels
 
+    @lru_cache()
     def get_build_script_files(self):
         """
         Dict of files to be copied to the container image for use in building.
@@ -324,6 +332,7 @@ class BuildPack:
                     f"Found a stencila manifest.xml at {root}. Stencila is no longer supported."
                 )
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Ordered list of shell script snippets to build the base image.
@@ -345,6 +354,7 @@ class BuildPack:
 
         return []
 
+    @lru_cache()
     def get_preassemble_script_files(self):
         """
         Dict of files to be copied to the container image for use in preassembly.
@@ -358,6 +368,7 @@ class BuildPack:
         """
         return {}
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         """
         Ordered list of shell snippets to build an image for this repository.
@@ -374,6 +385,7 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """
         Ordered list of shell script snippets to build the repo into the image.
@@ -400,6 +412,7 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_post_build_scripts(self):
         """
         An ordered list of executable scripts to execute after build.
@@ -412,6 +425,7 @@ class BuildPack:
         """
         return []
 
+    @lru_cache()
     def get_start_script(self):
         """
         The path to a script to be executed at container start up.
@@ -624,12 +638,14 @@ class BuildPack:
 
 
 class BaseImage(BuildPack):
+    @lru_cache()
     def get_build_env(self):
         """Return env directives required for build"""
         return [
             ("APP_BASE", "/srv"),
         ]
 
+    @lru_cache()
     def get_env(self):
         """Return env directives to be set after build"""
         return []
@@ -637,6 +653,7 @@ class BaseImage(BuildPack):
     def detect(self):
         return True
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         scripts = []
         try:
@@ -676,16 +693,19 @@ class BaseImage(BuildPack):
 
         return scripts
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """Return directives to run after the entire repository has been added to the image"""
         return []
 
+    @lru_cache()
     def get_post_build_scripts(self):
         post_build = self.binder_path("postBuild")
         if os.path.exists(post_build):
             return [post_build]
         return []
 
+    @lru_cache()
     def get_start_script(self):
         start = self.binder_path("start")
         if os.path.exists(start):

repo2docker/buildpacks/conda/__init__.py

@@ -3,6 +3,7 @@ import os
 import re
 import warnings
 from collections.abc import Mapping
+from functools import lru_cache
 
 from ruamel.yaml import YAML
 
@@ -45,6 +46,7 @@ class CondaBuildPack(BaseImage):
             return "linux-aarch64"
         raise ValueError(f"Unknown platform {self.platform}")
 
+    @lru_cache()
     def get_build_env(self):
         """Return environment variables to be set.
 
@@ -88,11 +90,13 @@ class CondaBuildPack(BaseImage):
             env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}"))
         return env
 
+    @lru_cache()
     def get_env(self):
         """Make kernel env the default for `conda install`"""
         env = super().get_env() + [("CONDA_DEFAULT_ENV", "${KERNEL_PYTHON_PREFIX}")]
         return env
 
+    @lru_cache()
     def get_path(self):
         """Return paths (including conda environment path) to be added to
         the PATH environment variable.
@@ -107,6 +111,7 @@ class CondaBuildPack(BaseImage):
         path.append("${NPM_DIR}/bin")
         return path
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Return series of build-steps common to all Python 3 repositories.
@@ -145,6 +150,7 @@ class CondaBuildPack(BaseImage):
 
     major_pythons = {"2": "2.7", "3": "3.10"}
 
+    @lru_cache()
     def get_build_script_files(self):
         """
         Dict of files to be copied to the container image for use in building.
@@ -369,6 +375,7 @@ class CondaBuildPack(BaseImage):
             self.kernel_env_cutoff_version
         )
 
+    @lru_cache()
     def get_preassemble_script_files(self):
         """preassembly only requires environment.yml
 
@@ -382,6 +389,7 @@ class CondaBuildPack(BaseImage):
                 assemble_files[environment_yml] = environment_yml
         return assemble_files
 
+    @lru_cache()
     def get_env_scripts(self):
         """Return series of build-steps specific to this source repository."""
         scripts = []
@@ -448,12 +456,14 @@ class CondaBuildPack(BaseImage):
             ]
         return scripts
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         scripts = super().get_preassemble_scripts()
         if self._should_preassemble_env:
             scripts.extend(self.get_env_scripts())
         return scripts
 
+    @lru_cache()
     def get_assemble_scripts(self):
         scripts = super().get_assemble_scripts()
         if not self._should_preassemble_env:

repo2docker/buildpacks/julia/julia_project.py

@@ -1,6 +1,7 @@
 """Generates a Dockerfile based on an input matrix for Julia"""
 import functools
 import os
+from functools import lru_cache
 
 import requests
 import semver
@@ -19,7 +20,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
     # Note that these must remain ordered, in order for the find_semver_match()
     # function to behave correctly.
     @property
-    @functools.lru_cache(maxsize=1)
+    @lru_cache(maxsize=1)
     def all_julias(self):
         try:
             json = requests.get(
@@ -67,6 +68,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
             raise RuntimeError("Failed to find a matching Julia version: {compat}")
         return match
 
+    @lru_cache()
     def get_build_env(self):
         """Get additional environment settings for Julia and Jupyter
 
@@ -109,9 +111,11 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
         else:
             return "${REPO_DIR}"
 
+    @lru_cache()
     def get_env(self):
         return super().get_env() + [("JULIA_PROJECT", self.project_dir)]
 
+    @lru_cache()
     def get_path(self):
         """Adds path to Julia binaries to user's PATH.
 
@@ -122,6 +126,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
         """
         return super().get_path() + ["${JULIA_PATH}/bin"]
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Return series of build-steps common to "ALL" Julia repositories
@@ -150,6 +155,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
             ),
         ]
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """
         Return series of build-steps specific to "this" Julia repository

repo2docker/buildpacks/julia/julia_require.py

@@ -1,6 +1,7 @@
 """Generates a Dockerfile based on an input matrix with REQUIRE for legacy Julia"""
 
 import os
+from functools import lru_cache
 
 from ...semver import parse_version as V
 from ..python import PythonBuildPack
@@ -54,6 +55,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
         self._julia_version = julia_version
         return self._julia_version
 
+    @lru_cache()
     def get_build_env(self):
         """Get additional environment settings for Julia and Jupyter
 
@@ -102,6 +104,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
             ("JUPYTER", jupyter),
         ]
 
+    @lru_cache()
     def get_path(self):
         """Adds path to Julia binaries to user's PATH.
 
@@ -112,6 +115,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
         """
         return super().get_path() + ["${JULIA_HOME}"]
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Return series of build-steps common to "ALL" Julia repositories
@@ -149,6 +153,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
             ),
         ]
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """
         Return series of build-steps specific to "this" Julia repository
@@ -176,6 +181,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
             )
         ]
 
+    @lru_cache()
     def get_build_script_files(self):
         files = {
             "julia/install-repo-dependencies.jl": "/tmp/install-repo-dependencies.jl"

repo2docker/buildpacks/nix/__init__.py

@@ -1,16 +1,19 @@
 """BuildPack for nixpkgs environments"""
 import os
+from functools import lru_cache
 
-from ..base import BaseImage, BuildPack
+from ..base import BaseImage
 
 
 class NixBuildPack(BaseImage):
     """A nix Package Manager BuildPack"""
 
+    @lru_cache()
     def get_path(self):
         """Return paths to be added to PATH environemnt variable"""
         return super().get_path() + ["/home/${NB_USER}/.nix-profile/bin"]
 
+    @lru_cache()
     def get_env(self):
         """Ordered list of environment variables to be set for this image"""
 
@@ -20,6 +23,7 @@ class NixBuildPack(BaseImage):
             ("GIT_SSL_CAINFO", "/etc/ssl/certs/ca-certificates.crt"),
         ]
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Return series of build-steps common to all nix repositories.
@@ -55,6 +59,7 @@ class NixBuildPack(BaseImage):
             ),
         ]
 
+    @lru_cache()
     def get_build_script_files(self):
         """Dict of files to be copied to the container image for use in building"""
         return {
@@ -62,6 +67,7 @@ class NixBuildPack(BaseImage):
             "nix/nix-shell-wrapper": "/usr/local/bin/nix-shell-wrapper",
         }
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """Return series of build-steps specific to this source repository."""
         return super().get_assemble_scripts() + [
@@ -75,6 +81,7 @@ class NixBuildPack(BaseImage):
             )
         ]
 
+    @lru_cache()
     def get_start_script(self):
         """The path to a script to be executed as ENTRYPOINT"""
         # the shell wrapper script duplicates the behaviour of other buildpacks

repo2docker/buildpacks/pipfile/__init__.py

@@ -8,6 +8,7 @@ same as the Python or Conda build packs.
 import json
 import os
 import re
+from functools import lru_cache
 
 import toml
 
@@ -79,6 +80,7 @@ class PipfileBuildPack(CondaBuildPack):
             )
             return self._python_version
 
+    @lru_cache()
     def get_preassemble_script_files(self):
         """Return files needed for preassembly"""
         files = super().get_preassemble_script_files()
@@ -88,6 +90,7 @@ class PipfileBuildPack(CondaBuildPack):
                 files[path] = path
         return files
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         """scripts to run prior to staging the repo contents"""
         scripts = super().get_preassemble_scripts()
@@ -105,6 +108,7 @@ class PipfileBuildPack(CondaBuildPack):
         )
         return scripts
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """Return series of build-steps specific to this repository."""
         # If we have either Pipfile.lock, Pipfile, or runtime.txt declare the

repo2docker/buildpacks/python/__init__.py

@@ -1,5 +1,6 @@
 """Generates Dockerfiles based on an input matrix based on Python."""
 import os
+from functools import lru_cache
 
 from ...utils import is_local_pip_requirement, open_guess_encoding
 from ..conda import CondaBuildPack
@@ -96,6 +97,7 @@ class PythonBuildPack(CondaBuildPack):
         # allow assembly from subset
         return True
 
+    @lru_cache()
     def get_preassemble_script_files(self):
         assemble_files = super().get_preassemble_script_files()
         for name in ("requirements.txt", "requirements3.txt"):
@@ -104,6 +106,7 @@ class PythonBuildPack(CondaBuildPack):
                 assemble_files[requirements_txt] = requirements_txt
         return assemble_files
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         """Return scripts to run before adding the full repository"""
         scripts = super().get_preassemble_scripts()
@@ -111,6 +114,7 @@ class PythonBuildPack(CondaBuildPack):
             scripts.extend(self._get_pip_scripts())
         return scripts
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """Return series of build steps that require the full repository"""
         # If we have a runtime.txt & that's set to python-2.7,

repo2docker/buildpacks/r.py

@@ -1,6 +1,7 @@
 import datetime
 import os
 import re
+from functools import lru_cache
 
 import requests
 
@@ -142,6 +143,7 @@ class RBuildPack(PythonBuildPack):
                 self._runtime = f"r-{str(self._checkpoint_date)}"
             return True
 
+    @lru_cache()
     def get_env(self):
         """
         Set custom env vars needed for RStudio to load
@@ -156,6 +158,7 @@ class RBuildPack(PythonBuildPack):
             ("LD_LIBRARY_PATH", "${R_HOME}/lib:${LD_LIBRARY_PATH}"),
         ]
 
+    @lru_cache()
     def get_path(self):
         """
         Return paths to be added to the PATH environment variable.
@@ -165,6 +168,7 @@ class RBuildPack(PythonBuildPack):
         """
         return super().get_path() + ["/usr/lib/rstudio-server/bin/"]
 
+    @lru_cache()
     def get_build_env(self):
         """
         Return environment variables to be set.
@@ -178,6 +182,7 @@ class RBuildPack(PythonBuildPack):
             ("R_LIBS_USER", "${APP_BASE}/rlibs")
         ]
 
+    @lru_cache()
     def get_packages(self):
         """
         Return list of packages to be installed.
@@ -195,6 +200,7 @@ class RBuildPack(PythonBuildPack):
 
         return super().get_packages().union(packages)
 
+    @lru_cache()
     def get_rspm_snapshot_url(self, snapshot_date, max_days_prior=7):
         for i in range(max_days_prior):
             snapshots = requests.post(
@@ -219,6 +225,7 @@ class RBuildPack(PythonBuildPack):
             )
         )
 
+    @lru_cache()
     def get_mran_snapshot_url(self, snapshot_date, max_days_prior=7):
         for i in range(max_days_prior):
             try_date = snapshot_date - datetime.timedelta(days=i)
@@ -233,6 +240,7 @@ class RBuildPack(PythonBuildPack):
             )
         )
 
+    @lru_cache()
     def get_cran_mirror_url(self, snapshot_date):
         # Date after which we will use rspm + binary packages instead of MRAN + source packages
         rspm_cutoff_date = datetime.date(2022, 1, 1)
@@ -242,6 +250,7 @@ class RBuildPack(PythonBuildPack):
         else:
             return self.get_mran_snapshot_url(snapshot_date)
 
+    @lru_cache()
     def get_devtools_snapshot_url(self):
         """
         Return url of snapshot to use for getting devtools install
@@ -255,6 +264,7 @@ class RBuildPack(PythonBuildPack):
         # necessary apt packages.
         return "https://packagemanager.rstudio.com/all/__linux__/bionic/2022-01-04+Y3JhbiwyOjQ1MjYyMTU7NzlBRkJEMzg"
 
+    @lru_cache()
     def get_build_scripts(self):
         """
         Return series of build-steps common to all R repositories
@@ -349,6 +359,7 @@ class RBuildPack(PythonBuildPack):
 
         return super().get_build_scripts() + scripts
 
+    @lru_cache()
     def get_preassemble_script_files(self):
         files = super().get_preassemble_script_files()
         installR_path = self.binder_path("install.R")
@@ -357,6 +368,7 @@ class RBuildPack(PythonBuildPack):
 
         return files
 
+    @lru_cache()
     def get_preassemble_scripts(self):
         """Install contents of install.R
 
@@ -382,6 +394,7 @@ class RBuildPack(PythonBuildPack):
 
         return super().get_preassemble_scripts() + scripts
 
+    @lru_cache()
     def get_assemble_scripts(self):
         """Install the dependencies of or the repository itself"""
         assemble_scripts = super().get_assemble_scripts()