From db8af0542080c25d4431dff30007b6903e21da01 Mon Sep 17 00:00:00 2001 From: ale Date: Thu, 20 Jun 2024 12:13:53 +0200 Subject: [PATCH] repaired Dockerfiles --- .gitlab-ci.yml | 41 +++++++++++++++++++++++++------------- docker/Dockerfile | 46 +++++++++++++++++++++++++++++++++++++++++++ docker/arm.Dockerfile | 44 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 docker/Dockerfile create mode 100644 docker/arm.Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4727288..fcb727c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,11 +6,22 @@ services: stages: - build_amd64 - build_arm64 - - build_cuda +# - build_cuda before_script: - docker -v +#build_amd64: +# stage: build_amd64 +# needs: [] +# before_script: +# - git submodule update --init --recursive +# script: +# - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - docker compose build libretranslate +# - docker tag libretranslate $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:latest +# - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:latest + build_amd64: stage: build_amd64 needs: [] @@ -18,8 +29,10 @@ build_amd64: - git submodule update --init --recursive script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker compose build libretranslate - - docker tag libretranslate $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:latest + - docker buildx build --build-arg "with_models=true" -t libretranslate:amd64 -f ./docker/Dockerfile LibreTranslate/ + - docker tag libretranslate:amd64 $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:amd64 + - docker tag libretranslate:amd64 $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:latest + - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:amd64 - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:latest build_arm64: @@ -30,17 +43,17 @@ build_arm64: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker buildx build --platform arm64 --build-arg "with_models=true" -t libretranslate:arm64 -f LibreTranslate/docker/arm.Dockerfile LibreTranslate/ + - docker buildx build --platform arm64 --build-arg "with_models=true" -t libretranslate:arm64 -f ./docker/arm.Dockerfile LibreTranslate/ - docker tag libretranslate:arm64 $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:arm64 - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:arm64 -build_cuda: - stage: build_cuda - needs: [] - before_script: - - git submodule update --init --recursive - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker buildx build --build-arg "with_models=true" -t libretranslate:cuda -f LibreTranslate/docker/cuda.Dockerfile LibreTranslate/ - - docker tag libretranslate:cuda $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:cuda - - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:cuda +#build_cuda: +# stage: build_cuda +# needs: [] +# before_script: +# - git submodule update --init --recursive +# script: +# - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - docker buildx build --build-arg "with_models=true" -t libretranslate:cuda -f LibreTranslate/docker/cuda.Dockerfile LibreTranslate/ +# - docker tag libretranslate:cuda $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:cuda +# - docker push $CI_REGISTRY/manalejandro/libretranslate-api/libretranslate-api:cuda diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..fbbb414 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,46 @@ +FROM python:3.11.9-slim-bullseye as builder + +WORKDIR /app + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update -qq \ + && apt-get -qqq install --no-install-recommends -y pkg-config gcc g++ \ + && apt-get upgrade --assume-yes \ + && apt-get clean \ + && rm -rf /var/lib/apt + +RUN python -mvenv venv && ./venv/bin/pip install --no-cache-dir --upgrade pip + +COPY . . + +# Install package from source code, compile translations +RUN ./venv/bin/pip install Babel==2.12.1 && ./venv/bin/python scripts/compile_locales.py \ + && ./venv/bin/pip install torch==2.0.1 --extra-index-url https://download.pytorch.org/whl/cpu \ + && ./venv/bin/pip install . \ + && ./venv/bin/pip cache purge + +FROM python:3.11.9-slim-bullseye + +ARG with_models=false +ARG models="" + +RUN addgroup --system --gid 1032 libretranslate && adduser --system --uid 1032 libretranslate && mkdir -p /home/libretranslate/.local && chown -R libretranslate:libretranslate /home/libretranslate/.local +USER libretranslate + +COPY --from=builder --chown=1032:1032 /app /app +WORKDIR /app + +COPY --from=builder --chown=1032:1032 /app/venv/bin/ltmanage /usr/bin/ + +RUN if [ "$with_models" = "true" ]; then \ + # initialize the language models + if [ ! -z "$models" ]; then \ + ./venv/bin/python scripts/install_models.py --load_only_lang_codes "$models"; \ + else \ + ./venv/bin/python scripts/install_models.py; \ + fi \ + fi + +RUN ./venv/bin/pip install numpy==1.24.1 +EXPOSE 5000 +ENTRYPOINT [ "./venv/bin/libretranslate", "--host", "*" ] diff --git a/docker/arm.Dockerfile b/docker/arm.Dockerfile new file mode 100644 index 0000000..9338c97 --- /dev/null +++ b/docker/arm.Dockerfile @@ -0,0 +1,44 @@ +FROM arm64v8/python:3.11.9-slim-bullseye as builder + +WORKDIR /app + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update -qq \ + && apt-get -qqq install --no-install-recommends -y pkg-config gcc g++ \ + && apt-get upgrade --assume-yes \ + && apt-get clean \ + && rm -rf /var/lib/apt + +RUN python -mvenv venv && ./venv/bin/pip install --no-cache-dir --upgrade pip + +COPY . . + +# Install package from source code, compile translations +RUN ./venv/bin/pip install Babel==2.12.1 && ./venv/bin/python scripts/compile_locales.py \ + && ./venv/bin/pip install torch==2.0.1 --extra-index-url https://download.pytorch.org/whl/cpu \ + && ./venv/bin/pip install . \ + && ./venv/bin/pip cache purge + +FROM arm64v8/python:3.11.9-slim-bullseye + +ARG with_models=false +ARG models="" + +RUN addgroup --system --gid 1032 libretranslate && adduser --system --uid 1032 libretranslate && mkdir -p /home/libretranslate/.local && chown -R libretranslate:libretranslate /home/libretranslate/.local +USER libretranslate + +COPY --from=builder --chown=1032:1032 /app /app +WORKDIR /app + +RUN if [ "$with_models" = "true" ]; then \ + # initialize the language models + if [ ! -z "$models" ]; then \ + ./venv/bin/python scripts/install_models.py --load_only_lang_codes "$models"; \ + else \ + ./venv/bin/python scripts/install_models.py; \ + fi \ + fi + +RUN ./venv/bin/pip install numpy==1.24.1 +EXPOSE 5000 +ENTRYPOINT [ "./venv/bin/libretranslate", "--host", "*" ]