From b590afa52adc1eb1a42164e0578278e5ab14b744 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 10:25:48 +0300 Subject: [PATCH 01/10] Refactoring of dockerfiles Let's to try using a pure dockerfile' logic. It is the fist step. - We use docker instead docker-composer - We upgrade and use only "std" plaform. Other plaforms will be upgraded later --- .travis.yml | 9 ++++----- Dockerfile--std.tmpl | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 997945b5..e24185e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,10 @@ services: - docker install: - - ./mk_dockerfile.sh - - docker-compose build + - docker build --build-arg PG_VERSION="${PG_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t tests -f Dockerfile--${TEST_PLATFORM}.tmpl . script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + - docker run $(bash <(curl -s https://codecov.io/env)) tests notifications: email: @@ -27,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 +# - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 +# - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index d844c9a3..6a9b7947 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -1,14 +1,16 @@ -FROM postgres:${PG_VERSION}-alpine +ARG PG_VERSION +ARG PYTHON_VERSION + +FROM postgres:${PG_VERSION}-alpine as base1 + +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip + +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv + +FROM base2_with_python-${PYTHON_VERSION} as final -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi ENV LANG=C.UTF-8 ADD . /pg/testgres @@ -16,4 +18,4 @@ WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh +ENTRYPOINT PYTHON_VERSION="${PYTHON_VERSION}" bash run_tests.sh From f161d41fe4372f93904bb4f747138cccf837dec3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 11:22:42 +0300 Subject: [PATCH 02/10] It is better to run a docker with "-t" options. This allows the colors in output data. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e24185e9..e4d83591 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - docker build --build-arg PG_VERSION="${PG_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t tests -f Dockerfile--${TEST_PLATFORM}.tmpl . script: - - docker run $(bash <(curl -s https://codecov.io/env)) tests + - docker run $(bash <(curl -s https://codecov.io/env)) -t tests notifications: email: From ce66524945414ab84f2dc9d2fd3ee63e38beee54 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 11:42:24 +0300 Subject: [PATCH 03/10] A problem with PYTHON_VERSION is fixed --- Dockerfile--std.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index 6a9b7947..ea8e676b 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -5,9 +5,11 @@ FROM postgres:${PG_VERSION}-alpine as base1 FROM base1 as base2_with_python-2 RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 FROM base2_with_python-${PYTHON_VERSION} as final @@ -18,4 +20,4 @@ WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION="${PYTHON_VERSION}" bash run_tests.sh +ENTRYPOINT bash run_tests.sh From 79fd6bafcaa59462dd55692e1d7e9f1d29c49914 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:13:29 +0300 Subject: [PATCH 04/10] Dockerfile--std.all.tmpl is updated --- .travis.yml | 2 +- Dockerfile--std.all.tmpl | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4d83591..9353d4a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 -# - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 # - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std.all.tmpl index dfd9ab20..c41c5a06 100644 --- a/Dockerfile--std.all.tmpl +++ b/Dockerfile--std.all.tmpl @@ -1,14 +1,21 @@ -FROM postgres:${PG_VERSION}-alpine - -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-2 +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final #RUN apk add --no-cache mc @@ -57,4 +64,4 @@ ls -la ~/.ssh/; \ sudo /usr/sbin/sshd; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash run_tests.sh;" From 64e4b84b2067bb340bd649d27d4a8527d8e98d4d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:17:05 +0300 Subject: [PATCH 05/10] Dockerfile--ubuntu-24_04.tmpl --- .travis.yml | 2 +- Dockerfile--ubuntu-24_04.tmpl | 48 +++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9353d4a8..29cb9afd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 -# - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu-24_04.tmpl index fd1136d8..d6a1d39c 100644 --- a/Dockerfile--ubuntu-24_04.tmpl +++ b/Dockerfile--ubuntu-24_04.tmpl @@ -1,7 +1,17 @@ -FROM ubuntu:24.04 +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM ubuntu:24.04 as base1 +ARG PG_VERSION + +RUN apt update +RUN apt install -y sudo curl ca-certificates +RUN apt update +RUN apt install -y openssh-server RUN apt update -RUN apt install -y sudo curl ca-certificates postgresql-common +RUN apt install -y postgresql-common RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y @@ -14,21 +24,12 @@ RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail http RUN apt update RUN apt install -y postgresql-${PG_VERSION} -RUN apt install -y python3 python3-dev python3-virtualenv # RUN apt install -y mc -# It is required for psycopg2 -RUN apt install -y libpq-dev -RUN apt install -y openssh-server - # [2025-02-26] It adds the user 'postgres' in the group 'sudo' # [2025-02-27] It is not required. # RUN adduser postgres sudo -ADD . /pg/testgres -WORKDIR /pg/testgres -RUN chown -R postgres /pg - EXPOSE 22 RUN ssh-keygen -A @@ -36,13 +37,32 @@ RUN ssh-keygen -A # It enables execution of "sudo service ssh start" without password RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers -USER postgres +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apt update +RUN apt install -y python3 +RUN apt update +RUN apt install -y python3-dev +RUN apt update +RUN apt install -y python3-virtualenv +RUN apt update +RUN apt install -y libpq-dev # It is required for psycopg2 +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + ENV LANG=C.UTF-8 +USER postgres + RUN chmod 700 ~/ RUN mkdir -p ~/.ssh -#ENTRYPOINT PYTHON_VERSION=3.12 /run.sh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ @@ -55,4 +75,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash ./run_tests.sh;" From 79bee5bc9b5fa2cfc6380adf776195d074718c5c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:19:01 +0300 Subject: [PATCH 06/10] Dockerfile--std.tmpl is updated (formatting) --- Dockerfile--std.tmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index ea8e676b..91886ede 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -1,16 +1,20 @@ ARG PG_VERSION ARG PYTHON_VERSION +# --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 +# --------------------------------------------- base2_with_python-2 FROM base1 as base2_with_python-2 RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip ENV PYTHON_VERSION=2 +# --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv ENV PYTHON_VERSION=3 +# --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} as final ENV LANG=C.UTF-8 From af6b694d3b78b9aad3b0c48a14dd97a03248a4bf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:30:24 +0300 Subject: [PATCH 07/10] docker-compose.sh and mk_dockerfile.yml are deleted --- docker-compose.yml | 4 ---- mk_dockerfile.sh | 2 -- 2 files changed, 6 deletions(-) delete mode 100644 docker-compose.yml delete mode 100755 mk_dockerfile.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 86edf9a4..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: '3.8' -services: - tests: - build: . diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh deleted file mode 100755 index 8f7876a3..00000000 --- a/mk_dockerfile.sh +++ /dev/null @@ -1,2 +0,0 @@ -set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile--${TEST_PLATFORM}.tmpl > Dockerfile From 4a7e875477b80cdfdd0a51bd3c0285041e8c95c0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:42:20 +0300 Subject: [PATCH 08/10] [CI] Platform "std.all" was renamed with "std_all" Let's avoid using a "point" symbol in name of file. This symbol may create a problem in the future if we decide to use configuration docker files without extensions. --- .travis.yml | 2 +- Dockerfile--std.all.tmpl => Dockerfile--std_all.tmpl | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename Dockerfile--std.all.tmpl => Dockerfile--std_all.tmpl (100%) diff --git a/.travis.yml b/.travis.yml index 29cb9afd..091b3192 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std_all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std_all.tmpl similarity index 100% rename from Dockerfile--std.all.tmpl rename to Dockerfile--std_all.tmpl From 2929e833fbb01597506c4a18414ca59677cc7584 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 17:52:02 +0300 Subject: [PATCH 09/10] [CI] Platform name has the one format Dockerfiles--[-].tmpl - "std_all" -> "std-all" - "ubuntu-24_04" -> "ubuntu_24_04" --- .travis.yml | 4 ++-- Dockerfile--std_all.tmpl => Dockerfile--std-all.tmpl | 0 ...erfile--ubuntu-24_04.tmpl => Dockerfile--ubuntu_24_04.tmpl | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename Dockerfile--std_all.tmpl => Dockerfile--std-all.tmpl (100%) rename Dockerfile--ubuntu-24_04.tmpl => Dockerfile--ubuntu_24_04.tmpl (100%) diff --git a/.travis.yml b/.travis.yml index 091b3192..3a889845 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - - TEST_PLATFORM=std_all PYTHON_VERSION=3 PG_VERSION=17 - - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std_all.tmpl b/Dockerfile--std-all.tmpl similarity index 100% rename from Dockerfile--std_all.tmpl rename to Dockerfile--std-all.tmpl diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl similarity index 100% rename from Dockerfile--ubuntu-24_04.tmpl rename to Dockerfile--ubuntu_24_04.tmpl From 3a7f475cedc6f50baa4c7acde19946f3ead5183a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 12 Mar 2025 20:01:24 +0300 Subject: [PATCH 10/10] Dockerfile--ubuntu_24_04.tmpl is updated (minimization) --- Dockerfile--ubuntu_24_04.tmpl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index d6a1d39c..c1ddeab6 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -39,14 +39,7 @@ RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoe # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 -RUN apt update -RUN apt install -y python3 -RUN apt update -RUN apt install -y python3-dev -RUN apt update -RUN apt install -y python3-virtualenv -RUN apt update -RUN apt install -y libpq-dev # It is required for psycopg2 +RUN apt install -y python3 python3-dev python3-virtualenv libpq-dev ENV PYTHON_VERSION=3 # --------------------------------------------- final