kopia lustrzana https://github.com/jupyterhub/repo2docker
Make platform configurable instead of only auto-detecting
rodzic
064d91ae22
commit
8f3f57ada7
|
@ -15,6 +15,7 @@ import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
import warnings
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import entrypoints
|
import entrypoints
|
||||||
|
@ -36,7 +37,7 @@ from .buildpacks import (
|
||||||
RBuildPack,
|
RBuildPack,
|
||||||
)
|
)
|
||||||
from .engine import BuildError, ContainerEngineException, ImageLoadError
|
from .engine import BuildError, ContainerEngineException, ImageLoadError
|
||||||
from .utils import ByteSpecification, R2dState, chdir
|
from .utils import ByteSpecification, R2dState, chdir, get_platform
|
||||||
|
|
||||||
|
|
||||||
class Repo2Docker(Application):
|
class Repo2Docker(Application):
|
||||||
|
@ -250,6 +251,27 @@ class Repo2Docker(Application):
|
||||||
config=True,
|
config=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
platform = Unicode(
|
||||||
|
config=True,
|
||||||
|
help="""
|
||||||
|
Platform to build for, linux/amd64 (recommended) or linux/arm64 (experimental).
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
@default("platform")
|
||||||
|
def _platform_default(self):
|
||||||
|
"""
|
||||||
|
Default platform
|
||||||
|
"""
|
||||||
|
p = get_platform()
|
||||||
|
if p == "linux/arm64":
|
||||||
|
warnings.warn(
|
||||||
|
"Building for linux/arm64 is experimental. "
|
||||||
|
"To use the recommended platform set --Repo2Docker.platform=linux/amd64. "
|
||||||
|
"To silence this warning set --Repo2Docker.platform=linux/arm64."
|
||||||
|
)
|
||||||
|
return p
|
||||||
|
|
||||||
extra_build_args = Dict(
|
extra_build_args = Dict(
|
||||||
{},
|
{},
|
||||||
help="""
|
help="""
|
||||||
|
@ -778,6 +800,7 @@ class Repo2Docker(Application):
|
||||||
else:
|
else:
|
||||||
picked_buildpack = self.default_buildpack()
|
picked_buildpack = self.default_buildpack()
|
||||||
|
|
||||||
|
picked_buildpack.platform = self.platform
|
||||||
picked_buildpack.appendix = self.appendix
|
picked_buildpack.appendix = self.appendix
|
||||||
# Add metadata labels
|
# Add metadata labels
|
||||||
picked_buildpack.labels["repo2docker.version"] = self.version
|
picked_buildpack.labels["repo2docker.version"] = self.version
|
||||||
|
@ -819,6 +842,7 @@ class Repo2Docker(Application):
|
||||||
build_args,
|
build_args,
|
||||||
self.cache_from,
|
self.cache_from,
|
||||||
self.extra_build_kwargs,
|
self.extra_build_kwargs,
|
||||||
|
platform=self.platform,
|
||||||
):
|
):
|
||||||
if docker_client.string_output:
|
if docker_client.string_output:
|
||||||
self.log.info(l, extra=dict(phase=R2dState.BUILDING))
|
self.log.info(l, extra=dict(phase=R2dState.BUILDING))
|
||||||
|
|
|
@ -4,15 +4,11 @@ Base information for using R in BuildPacks.
|
||||||
Keeping this in r.py would lead to cyclic imports.
|
Keeping this in r.py would lead to cyclic imports.
|
||||||
"""
|
"""
|
||||||
from ..semver import parse_version as V
|
from ..semver import parse_version as V
|
||||||
from ..utils import get_platform
|
|
||||||
|
|
||||||
|
|
||||||
def rstudio_base_scripts(r_version):
|
def rstudio_base_scripts(r_version):
|
||||||
"""Base steps to install RStudio and shiny-server."""
|
"""Base steps to install RStudio and shiny-server."""
|
||||||
|
|
||||||
if get_platform() != "linux-64":
|
|
||||||
raise RuntimeError("RStudio is only available for linux-64")
|
|
||||||
|
|
||||||
# Shiny server (not the package!) seems to be the same version for all R versions
|
# Shiny server (not the package!) seems to be the same version for all R versions
|
||||||
shiny_server_url = "https://download3.rstudio.org/ubuntu-14.04/x86_64/shiny-server-1.5.17.973-amd64.deb"
|
shiny_server_url = "https://download3.rstudio.org/ubuntu-14.04/x86_64/shiny-server-1.5.17.973-amd64.deb"
|
||||||
shiny_proxy_version = "1.1"
|
shiny_proxy_version = "1.1"
|
||||||
|
|
|
@ -11,8 +11,6 @@ import textwrap
|
||||||
import escapism
|
import escapism
|
||||||
import jinja2
|
import jinja2
|
||||||
|
|
||||||
from ..utils import get_platform
|
|
||||||
|
|
||||||
# Only use syntax features supported by Docker 17.09
|
# Only use syntax features supported by Docker 17.09
|
||||||
TEMPLATE = r"""
|
TEMPLATE = r"""
|
||||||
FROM buildpack-deps:bionic
|
FROM buildpack-deps:bionic
|
||||||
|
@ -231,7 +229,7 @@ class BuildPack:
|
||||||
"Windows environment detected. Note that Windows "
|
"Windows environment detected. Note that Windows "
|
||||||
"support is experimental in repo2docker."
|
"support is experimental in repo2docker."
|
||||||
)
|
)
|
||||||
self.platform = get_platform()
|
self.platform = ""
|
||||||
|
|
||||||
def get_packages(self):
|
def get_packages(self):
|
||||||
"""
|
"""
|
||||||
|
@ -562,6 +560,7 @@ class BuildPack:
|
||||||
build_args,
|
build_args,
|
||||||
cache_from,
|
cache_from,
|
||||||
extra_build_kwargs,
|
extra_build_kwargs,
|
||||||
|
platform=None,
|
||||||
):
|
):
|
||||||
tarf = io.BytesIO()
|
tarf = io.BytesIO()
|
||||||
tar = tarfile.open(fileobj=tarf, mode="w")
|
tar = tarfile.open(fileobj=tarf, mode="w")
|
||||||
|
@ -616,6 +615,7 @@ class BuildPack:
|
||||||
buildargs=build_args,
|
buildargs=build_args,
|
||||||
container_limits=limits,
|
container_limits=limits,
|
||||||
cache_from=cache_from,
|
cache_from=cache_from,
|
||||||
|
platform=platform,
|
||||||
)
|
)
|
||||||
|
|
||||||
build_kwargs.update(extra_build_kwargs)
|
build_kwargs.update(extra_build_kwargs)
|
||||||
|
|
|
@ -35,6 +35,14 @@ class CondaBuildPack(BaseImage):
|
||||||
# extra pip requirements.txt for the notebook env
|
# extra pip requirements.txt for the notebook env
|
||||||
_nb_requirements_file = ""
|
_nb_requirements_file = ""
|
||||||
|
|
||||||
|
def _conda_platform(self):
|
||||||
|
"""Return the conda platform name for the current platform"""
|
||||||
|
if self.platform == "linux/amd64":
|
||||||
|
return "linux-64"
|
||||||
|
if self.platform == "linux/arm64":
|
||||||
|
return "linux-aarch64"
|
||||||
|
raise ValueError(f"Unknown platform {self.platform}")
|
||||||
|
|
||||||
def get_build_env(self):
|
def get_build_env(self):
|
||||||
"""Return environment variables to be set.
|
"""Return environment variables to be set.
|
||||||
|
|
||||||
|
@ -59,7 +67,7 @@ class CondaBuildPack(BaseImage):
|
||||||
# this exe should be used for installs after bootstrap with micromamba
|
# this exe should be used for installs after bootstrap with micromamba
|
||||||
# switch this to /usr/local/bin/micromamba to use it for all installs
|
# switch this to /usr/local/bin/micromamba to use it for all installs
|
||||||
("MAMBA_EXE", "${CONDA_DIR}/bin/mamba"),
|
("MAMBA_EXE", "${CONDA_DIR}/bin/mamba"),
|
||||||
("CONDA_PLATFORM", self.platform),
|
("CONDA_PLATFORM", self._conda_platform()),
|
||||||
]
|
]
|
||||||
if self._nb_requirements_file:
|
if self._nb_requirements_file:
|
||||||
env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file))
|
env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file))
|
||||||
|
@ -161,7 +169,7 @@ class CondaBuildPack(BaseImage):
|
||||||
# major Python versions during upgrade.
|
# major Python versions during upgrade.
|
||||||
# If no version is specified or no matching X.Y version is found,
|
# If no version is specified or no matching X.Y version is found,
|
||||||
# the default base environment is used.
|
# the default base environment is used.
|
||||||
frozen_name = f"environment-{self.platform}.lock"
|
frozen_name = f"environment-{self._conda_platform()}.lock"
|
||||||
pip_frozen_name = "requirements.txt"
|
pip_frozen_name = "requirements.txt"
|
||||||
if py_version:
|
if py_version:
|
||||||
if self.python_version == "2.7":
|
if self.python_version == "2.7":
|
||||||
|
@ -177,13 +185,15 @@ class CondaBuildPack(BaseImage):
|
||||||
self._kernel_requirements_file
|
self._kernel_requirements_file
|
||||||
) = "/tmp/env/kernel-requirements.txt"
|
) = "/tmp/env/kernel-requirements.txt"
|
||||||
else:
|
else:
|
||||||
py_frozen_name = f"environment.py-{py_version}-{self.platform}.lock"
|
py_frozen_name = (
|
||||||
|
f"environment.py-{py_version}-{self._conda_platform()}.lock"
|
||||||
|
)
|
||||||
if os.path.exists(os.path.join(HERE, py_frozen_name)):
|
if os.path.exists(os.path.join(HERE, py_frozen_name)):
|
||||||
frozen_name = py_frozen_name
|
frozen_name = py_frozen_name
|
||||||
pip_frozen_name = f"requirements.py-{py_version}.pip"
|
pip_frozen_name = f"requirements.py-{py_version}.pip"
|
||||||
else:
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Python version {py_version} {self.platform} is not supported!"
|
f"Python version {py_version} {self._conda_platform()} is not supported!"
|
||||||
)
|
)
|
||||||
files[
|
files[
|
||||||
"conda/" + frozen_name
|
"conda/" + frozen_name
|
||||||
|
@ -369,6 +379,10 @@ class CondaBuildPack(BaseImage):
|
||||||
""",
|
""",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if self.platform != "linux/amd64":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"RStudio is only available for linux/amd64 ({self.platform})"
|
||||||
|
)
|
||||||
scripts += rstudio_base_scripts(self.r_version)
|
scripts += rstudio_base_scripts(self.r_version)
|
||||||
scripts += [
|
scripts += [
|
||||||
(
|
(
|
||||||
|
|
|
@ -30,6 +30,7 @@ class DockerBuildPack(BuildPack):
|
||||||
build_args,
|
build_args,
|
||||||
cache_from,
|
cache_from,
|
||||||
extra_build_kwargs,
|
extra_build_kwargs,
|
||||||
|
platform=None,
|
||||||
):
|
):
|
||||||
"""Build a Docker image based on the Dockerfile in the source repo."""
|
"""Build a Docker image based on the Dockerfile in the source repo."""
|
||||||
# If you work on this bit of code check the corresponding code in
|
# If you work on this bit of code check the corresponding code in
|
||||||
|
@ -55,6 +56,7 @@ class DockerBuildPack(BuildPack):
|
||||||
container_limits=limits,
|
container_limits=limits,
|
||||||
cache_from=cache_from,
|
cache_from=cache_from,
|
||||||
labels=self.get_labels(),
|
labels=self.get_labels(),
|
||||||
|
platform=platform,
|
||||||
)
|
)
|
||||||
|
|
||||||
build_kwargs.update(extra_build_kwargs)
|
build_kwargs.update(extra_build_kwargs)
|
||||||
|
|
|
@ -87,7 +87,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack):
|
||||||
For example, a tuple may be `('JULIA_VERSION', '0.6.0')`.
|
For example, a tuple may be `('JULIA_VERSION', '0.6.0')`.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if self.platform == "linux-aarch64":
|
if self.platform == "linux/arm64":
|
||||||
julia_arch = julia_arch_short = "aarch64"
|
julia_arch = julia_arch_short = "aarch64"
|
||||||
else:
|
else:
|
||||||
julia_arch = "x86_64"
|
julia_arch = "x86_64"
|
||||||
|
|
|
@ -77,7 +77,7 @@ class JuliaRequireBuildPack(PythonBuildPack):
|
||||||
For example, a tuple may be `('JULIA_VERSION', '0.6.0')`.
|
For example, a tuple may be `('JULIA_VERSION', '0.6.0')`.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if self.platform == "linux-aarch64":
|
if self.platform == "linux/arm64":
|
||||||
julia_arch = julia_arch_short = "aarch64"
|
julia_arch = julia_arch_short = "aarch64"
|
||||||
else:
|
else:
|
||||||
julia_arch = "x86_64"
|
julia_arch = "x86_64"
|
||||||
|
|
|
@ -31,7 +31,7 @@ class NixBuildPack(BaseImage):
|
||||||
- install nix package manager for user
|
- install nix package manager for user
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if self.platform == "linux-aarch64":
|
if self.platform == "linux/arm64":
|
||||||
nix_arch = "aarch64"
|
nix_arch = "aarch64"
|
||||||
else:
|
else:
|
||||||
nix_arch = "x86_64"
|
nix_arch = "x86_64"
|
||||||
|
|
|
@ -5,7 +5,6 @@ import re
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from ..semver import parse_version as V
|
from ..semver import parse_version as V
|
||||||
from ..utils import get_platform
|
|
||||||
from ._r_base import rstudio_base_scripts
|
from ._r_base import rstudio_base_scripts
|
||||||
from .python import PythonBuildPack
|
from .python import PythonBuildPack
|
||||||
|
|
||||||
|
@ -278,8 +277,10 @@ class RBuildPack(PythonBuildPack):
|
||||||
|
|
||||||
cran_mirror_url = self.get_cran_mirror_url(self.checkpoint_date)
|
cran_mirror_url = self.get_cran_mirror_url(self.checkpoint_date)
|
||||||
|
|
||||||
if get_platform() != "linux-64":
|
if self.platform != "linux/amd64":
|
||||||
raise RuntimeError("RStudio is only available for linux-64")
|
raise RuntimeError(
|
||||||
|
f"RStudio is only available for linux/amd64 ({self.platform})"
|
||||||
|
)
|
||||||
scripts = [
|
scripts = [
|
||||||
(
|
(
|
||||||
"root",
|
"root",
|
||||||
|
|
|
@ -8,7 +8,6 @@ from traitlets import Dict
|
||||||
import docker
|
import docker
|
||||||
|
|
||||||
from .engine import Container, ContainerEngine, ContainerEngineException, Image
|
from .engine import Container, ContainerEngine, ContainerEngineException, Image
|
||||||
from .utils import get_platform
|
|
||||||
|
|
||||||
|
|
||||||
class DockerContainer(Container):
|
class DockerContainer(Container):
|
||||||
|
@ -92,14 +91,9 @@ class DockerEngine(ContainerEngine):
|
||||||
fileobj=None,
|
fileobj=None,
|
||||||
path="",
|
path="",
|
||||||
labels=None,
|
labels=None,
|
||||||
|
platform=None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
platform = get_platform()
|
|
||||||
if platform == "linux-aarch64":
|
|
||||||
docker_platform = "linux/arm64"
|
|
||||||
else:
|
|
||||||
docker_platform = "linux/amd64"
|
|
||||||
|
|
||||||
return self._apiclient.build(
|
return self._apiclient.build(
|
||||||
buildargs=buildargs,
|
buildargs=buildargs,
|
||||||
cache_from=cache_from,
|
cache_from=cache_from,
|
||||||
|
@ -113,7 +107,7 @@ class DockerEngine(ContainerEngine):
|
||||||
fileobj=fileobj,
|
fileobj=fileobj,
|
||||||
path=path,
|
path=path,
|
||||||
labels=labels,
|
labels=labels,
|
||||||
platform=docker_platform,
|
platform=platform,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -177,6 +177,7 @@ class ContainerEngine(LoggingConfigurable):
|
||||||
fileobj=None,
|
fileobj=None,
|
||||||
path="",
|
path="",
|
||||||
labels=None,
|
labels=None,
|
||||||
|
platform=None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
|
@ -207,6 +208,8 @@ class ContainerEngine(LoggingConfigurable):
|
||||||
path to the Dockerfile
|
path to the Dockerfile
|
||||||
labels : dict
|
labels : dict
|
||||||
Dictionary of labels to set on the image
|
Dictionary of labels to set on the image
|
||||||
|
platform: str
|
||||||
|
Platform to build for
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
|
|
|
@ -529,15 +529,15 @@ def is_local_pip_requirement(line):
|
||||||
|
|
||||||
|
|
||||||
def get_platform():
|
def get_platform():
|
||||||
"""Return the platform of the image
|
"""Return the target platform of the container image
|
||||||
|
|
||||||
Returns either `linux-64` or `linux-aarch64`
|
Returns either `linux/amd64` or `linux/arm64`
|
||||||
"""
|
"""
|
||||||
m = platform.machine()
|
m = platform.machine()
|
||||||
if m == "x86_64":
|
if m == "x86_64":
|
||||||
return "linux-64"
|
return "linux/amd64"
|
||||||
elif m == "aarch64":
|
elif m == "aarch64":
|
||||||
return "linux-aarch64"
|
return "linux/arm64"
|
||||||
else:
|
else:
|
||||||
warnings.warn(f"Unexpected platform '{m}', defaulting to linux-64")
|
warnings.warn(f"Unexpected platform '{m}', defaulting to linux/amd64")
|
||||||
return "linux-64"
|
return "linux/amd64"
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
Tests for repo2docker/utils.py
|
Tests for repo2docker/utils.py
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
|
import platform
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
@ -160,3 +161,16 @@ def test_open_guess_encoding():
|
||||||
)
|
)
|
||||||
def test_local_pip_requirement(req, is_local):
|
def test_local_pip_requirement(req, is_local):
|
||||||
assert utils.is_local_pip_requirement(req) == is_local
|
assert utils.is_local_pip_requirement(req) == is_local
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"machine_name,expected",
|
||||||
|
[
|
||||||
|
("x86_64", "linux/amd64"),
|
||||||
|
("aarch64", "linux/arm64"),
|
||||||
|
("other", "linux/amd64"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_get_platform(monkeypatch, machine_name, expected):
|
||||||
|
monkeypatch.setattr(platform, "machine", lambda: machine_name)
|
||||||
|
assert utils.get_platform() == expected
|
||||||
|
|
Ładowanie…
Reference in New Issue