Remove apk go requirement when possible (#2318)

* Install revive from ghcr docker binary

* Install actionlint from docker image's binary to remove apk go

* Install checkmake from docker image binary

* Use latest instead of tags for modified docker images

* Use a builder image for revive as the released binaries return broken version number

* Edit comment in actionlint Dockerfile instructions

* Build dustilock in a stage and copy go binary

* Specify GOBIN folder for revive install in builder

* Remove unneeded GOPATH and GOROOT variables

Unneeded since we are not building go packages from source anymore

* Include fixes from linters

* Add `COPY --link --from` by using dockerfile frontend syntax

* Revert: Remove unneeded GOPATH and GOROOT variables

* Run build.sh

* Run ./build.sh after rebase

* Include apk go requirement for golangci-lint

* Add changelog entry

* Handle duplicated FROM and COPY instructions

* cSpell fixes

Author: echoix

.automation/build.py

@@ -212,8 +212,13 @@ def generate_flavor(flavor, flavor_info):
             os.makedirs(os.path.dirname(dockerfile), exist_ok=True)
         copyfile(f"{REPO_HOME}/Dockerfile", dockerfile)
         flavor_label = flavor_info["label"]
-        comment = f"# MEGA-LINTER FLAVOR [{flavor}]: {flavor_label}"
+        comment = f"# MEGALINTER FLAVOR [{flavor}]: {flavor_label}"
         with open(dockerfile, "r+", encoding="utf-8") as f:
+            first_line = f.readline().rstrip()
+            if first_line.startswith("# syntax="):
+                comment = f"{first_line}\n{comment}"
+            else:
+                f.seek(0)
             content = f.read()
             f.seek(0)
             f.truncate()
@@ -300,14 +305,27 @@ def build_dockerfile(
             for dockerfile_item in item["install"]["dockerfile"]:
                 # FROM
                 if dockerfile_item.startswith("FROM"):
+                    if dockerfile_item in all_dockerfile_items:
+                        dockerfile_item = (
+                            "# Next FROM line commented because already managed by another linter\n"
+                            "# " + "\n# ".join(dockerfile_item.splitlines())
+                        )
                     docker_from += [dockerfile_item]
                 # ARG
                 elif dockerfile_item.startswith("ARG"):
                     docker_arg += [dockerfile_item]
                 # COPY
                 elif dockerfile_item.startswith("COPY"):
+                    if dockerfile_item in all_dockerfile_items:
+                        dockerfile_item = (
+                            "# Next COPY line commented because already managed by another linter\n"
+                            "# " + "\n# ".join(dockerfile_item.splitlines())
+                        )
                     docker_copy += [dockerfile_item]
-                    docker_other += ["# Managed with " + dockerfile_item]
+                    docker_other += [
+                        "# Managed with "
+                        + "\n#              ".join(dockerfile_item.splitlines())
+                    ]
                 # Already used item
                 elif (
                     dockerfile_item in all_dockerfile_items

.github/linters/.cspell.json

@@ -132,6 +132,7 @@
         "Fmegalinter",
         "GHSA",
         "GITLEAKS",
+        "GOBIN",
         "GOLANGCI",
         "GOODCHECK",
         "GOPATH",
@@ -539,6 +540,7 @@
         "descriptionless",
         "devcontainer",
         "devcontainers",
+        "devel",
         "devskim",
         "dimen",
         "dimendef",
@@ -864,6 +866,7 @@
         "mlinters",
         "moarvm",
         "mprettier",
+        "mrtazz",
         "msgon",
         "mstruebing",
         "msvs",
@@ -977,6 +980,7 @@
         "packagename",
         "pagebreak",
         "pageref",
+        "pandoc",
         "parallelization",
         "paren",
         "parens",

CHANGELOG.md

@@ -36,6 +36,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l
 - CI
   - Use docker/build-push-action to build docker images and akhilerm/tag-push-action to release by retagging and pushing beta images instead of rebuilding them
   - Authenticate to GitHub API during docker build to avoid reaching limits
+  - Remove apk go package install in images where possible to decrease image sizes, by @echoix in <https://github.com/oxsecurity/megalinter/pull/2318>
 
 - Fixes
   - Replace deprecated spectral package, by @bdovaz in by @bdovaz in <https://github.com/oxsecurity/megalinter/pull/2340>

Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 ###########################################
 ###########################################
 ## Dockerfile to run MegaLinter ##
@@ -10,15 +11,33 @@
 ## @generated by .automation/build.py using descriptor files, please do not update manually ##
 #############################################################################################
 #FROM__START
+FROM rhysd/actionlint:latest as actionlint
+# shellcheck is a dependency for actionlint
+
+FROM koalaman/shellcheck:stable as shellcheck
+# Next FROM line commented because already managed by another linter
+# FROM koalaman/shellcheck:stable as shellcheck
 FROM mvdan/shfmt:latest-alpine as shfmt
 FROM hadolint/hadolint:v2.12.0-alpine as hadolint
 FROM mstruebing/editorconfig-checker:2.7.0 as editorconfig-checker
+FROM golang:1.19-alpine as revive
+## The golang image used as a builder is a temporary workaround 
+## for the released revive binaries not returning version numbers (devel). 
+## The install command should then be what is commented in the go.megalinter-descriptor.yml
+RUN GOBIN=/usr/bin go install github.com/mgechev/revive@latest
+
 FROM ghcr.io/assignuser/chktex-alpine:latest as chktex
+FROM mrtazz/checkmake:latest as checkmake
 FROM yoheimuta/protolint:latest as protolint
+FROM golang:alpine as dustilock
+RUN GOBIN=/usr/bin go install github.com/checkmarx/[email protected]
+
 FROM zricethezav/gitleaks:v8.15.3 as gitleaks
 FROM ghcr.io/terraform-linters/tflint:v0.45.0 as tflint
 FROM tenable/terrascan:1.18.0 as terrascan
 FROM alpine/terragrunt:latest as terragrunt
+# Next FROM line commented because already managed by another linter
+# FROM alpine/terragrunt:latest as terragrunt
 FROM checkmarx/kics:alpine as kics
 #FROM__END
 
@@ -69,7 +88,6 @@ RUN apk add --update --no-cache \
                 openssh \
                 docker \
                 openrc \
-                go \
                 icu-libs \
                 libcurl \
                 libintl \
@@ -93,9 +111,11 @@ RUN apk add --update --no-cache \
                 php81-simplexml \
                 composer \
                 dpkg \
+                py3-pyflakes \
                 nodejs \
                 npm \
                 yarn \
+                go \
                 openssl \
                 readline-dev \
                 g++ \
@@ -280,17 +300,26 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 #############################################################################################
 
 #COPY__START
-COPY --from=shfmt /bin/shfmt /usr/bin/
-COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
-COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
-COPY --from=chktex /usr/bin/chktex /usr/bin/
-COPY --from=protolint /usr/local/bin/protolint /usr/bin/
-COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
-COPY --from=tflint /usr/local/bin/tflint /usr/bin/
-COPY --from=terrascan /go/bin/terrascan /usr/bin/
-COPY --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
-COPY --from=terragrunt /bin/terraform /usr/bin/
-COPY --from=kics /app/bin/kics /usr/bin/
+COPY --link --from=actionlint /usr/local/bin/actionlint /usr/bin/actionlint
+# shellcheck is a dependency for actionlint
+
+COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
+# Next COPY line commented because already managed by another linter
+# COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
+COPY --link --from=shfmt /bin/shfmt /usr/bin/
+COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
+COPY --link --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
+COPY --link --from=revive /usr/bin/revive /usr/bin/revive
+COPY --link --from=chktex /usr/bin/chktex /usr/bin/
+COPY --link --from=checkmake /checkmake /usr/bin/checkmake
+COPY --link --from=protolint /usr/local/bin/protolint /usr/bin/
+COPY --link --from=dustilock /usr/bin/dustilock /usr/bin/dustilock
+COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
+COPY --link --from=tflint /usr/local/bin/tflint /usr/bin/
+COPY --link --from=terrascan /go/bin/terrascan /usr/bin/
+COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
+COPY --link --from=terragrunt /bin/terraform /usr/bin/
+COPY --link --from=kics /app/bin/kics /usr/bin/
 COPY --from=kics /app/bin/assets /opt/kics/assets/
 #COPY__END
 
@@ -380,8 +409,9 @@ RUN echo y|sfdx plugins:install sfdx-hardis \
 # ENV PATH="${PATH}:/root/.dotnet/tools:/usr/share/dotnet"
 
 # actionlint installation
-ENV GO111MODULE=on
-RUN go install github.com/rhysd/actionlint/cmd/actionlint@latest && go clean --cache
+# Managed with COPY --link --from=actionlint /usr/local/bin/actionlint /usr/bin/actionlint
+#              # shellcheck is a dependency for actionlint
+# Managed with COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
 
 # arm-ttk installation
 ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1"
@@ -396,14 +426,11 @@ RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \
     && chmod +x /usr/bin/bash-exec \
 
 # shellcheck installation
-    && ML_THIRD_PARTY_DIR="/third-party/shellcheck" \
-    && mkdir -p ${ML_THIRD_PARTY_DIR} \
-    && wget -qO- "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv --directory ${ML_THIRD_PARTY_DIR} \
-    && mv "${ML_THIRD_PARTY_DIR}/shellcheck-stable/shellcheck" /usr/bin/ \
-    && find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \
+# Managed with # Next COPY line commented because already managed by another linter
+#              # COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
 
 # shfmt installation
-# Managed with COPY --from=shfmt /bin/shfmt /usr/bin/
+# Managed with COPY --link --from=shfmt /bin/shfmt /usr/bin/
 
 # bicep_linter installation
     && curl --retry 5 --retry-delay 5 -sLo ${BICEP_EXE} "${BICEP_URI}" \
@@ -428,20 +455,21 @@ RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \
     && rm -r dart-sdk/ \
 
 # hadolint installation
-# Managed with COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
+# Managed with COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
 
 # editorconfig-checker installation
-# Managed with COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
+# Managed with COPY --link --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
 
 # dotenv-linter installation
     && wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \
 
 # golangci-lint installation
     && wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh \
-    && golangci-lint --version \
+    && golangci-lint --version
+
 
 # revive installation
-    && go install github.com/mgechev/revive@latest && go clean --cache
+# Managed with COPY --link --from=revive /usr/bin/revive /usr/bin/revive
 
 # checkstyle installation
 RUN --mount=type=secret,id=GITHUB_TOKEN CHECKSTYLE_LATEST=$(curl -s \
@@ -487,7 +515,7 @@ RUN wget --quiet https://github.com/pmd/pmd/releases/download/pmd_releases%2F${P
     && find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \
 
 # chktex installation
-# Managed with COPY --from=chktex /usr/bin/chktex /usr/bin/
+# Managed with COPY --link --from=chktex /usr/bin/chktex /usr/bin/
     && cd ~ && touch .chktexrc && cd / \
 
 # luacheck installation
@@ -506,9 +534,7 @@ RUN wget --quiet https://github.com/pmd/pmd/releases/download/pmd_releases%2F${P
     && cd / \
 
 # checkmake installation
-    && ( [ -d /usr/local/bin ] || mkdir -p /usr/local/bin ) \
-    && wget -q "https://github.com/mrtazz/checkmake/releases/download/0.2.1/checkmake-0.2.1.linux.amd64" -O /usr/local/bin/checkmake \
-    && chmod 755 /usr/local/bin/checkmake \
+# Managed with COPY --link --from=checkmake /checkmake /usr/bin/checkmake
 
 # perlcritic installation
     && curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic
@@ -537,7 +563,7 @@ RUN composer global require --ignore-platform-reqs overtrue/phplint ^5.3 \
 # RUN pwsh -c 'Install-Module -Name PSScriptAnalyzer -RequiredVersion ${PSSA_VERSION} -Scope AllUsers -Force'
 
 # protolint installation
-# Managed with COPY --from=protolint /usr/local/bin/protolint /usr/bin/
+# Managed with COPY --link --from=protolint /usr/local/bin/protolint /usr/bin/
 
 # lintr installation
     && mkdir -p /home/r-library \
@@ -565,18 +591,10 @@ ENV PATH="~/.raku/bin:/opt/rakudo-pkg/bin:/opt/rakudo-pkg/share/perl6/site/bin:$
 RUN dotnet tool install --global Microsoft.CST.DevSkim.CLI \
 
 # dustilock installation
-    && ML_THIRD_PARTY_DIR=/download/dustilock && \
-    mkdir -p ${ML_THIRD_PARTY_DIR} && \
-    git clone https://github.com/Checkmarx/dustilock.git ${ML_THIRD_PARTY_DIR} && \
-    cd ${ML_THIRD_PARTY_DIR} && \
-    go build && go clean --cache && \
-    chmod +x dustilock && \
-    mv "${ML_THIRD_PARTY_DIR}/dustilock" /usr/bin/ && \
-    find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete && \
-    cd / \
+# Managed with COPY --link --from=dustilock /usr/bin/dustilock /usr/bin/dustilock
 
 # gitleaks installation
-# Managed with COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
+# Managed with COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
 
 # syft installation
     && curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin \
@@ -622,19 +640,19 @@ RUN dotnet tool install --global Microsoft.CST.DevSkim.CLI \
     && dotnet tool install --global TSQLLint \
 
 # tflint installation
-# Managed with COPY --from=tflint /usr/local/bin/tflint /usr/bin/
+# Managed with COPY --link --from=tflint /usr/local/bin/tflint /usr/bin/
 
 # terrascan installation
-# Managed with COPY --from=terrascan /go/bin/terrascan /usr/bin/
+# Managed with COPY --link --from=terrascan /go/bin/terrascan /usr/bin/
 
 # terragrunt installation
-# Managed with COPY --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
+# Managed with COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
 
 # terraform-fmt installation
-# Managed with COPY --from=terragrunt /bin/terraform /usr/bin/
+# Managed with COPY --link --from=terragrunt /bin/terraform /usr/bin/
 
 # kics installation
-# Managed with COPY --from=kics /app/bin/kics /usr/bin/
+# Managed with COPY --link --from=kics /app/bin/kics /usr/bin/
     && mkdir -p /opt/kics/assets
 ENV KICS_QUERIES_PATH=/opt/kics/assets/queries KICS_LIBRARIES_PATH=/opt/kics/assets/libraries
 # Managed with COPY --from=kics /app/bin/assets /opt/kics/assets/

flavors/ci_light/Dockerfile

@@ -1,4 +1,5 @@
-# MEGA-LINTER FLAVOR [ci_light]: Optimized for CI items (Dockerfile, Jenkinsfile, JSON/YAML schemas,XML
+# syntax=docker/dockerfile:1
+# MEGALINTER FLAVOR [ci_light]: Optimized for CI items (Dockerfile, Jenkinsfile, JSON/YAML schemas,XML
 ###########################################
 ###########################################
 ## Dockerfile to run MegaLinter ##
@@ -11,8 +12,10 @@
 ## @generated by .automation/build.py using descriptor files, please do not update manually ##
 #############################################################################################
 #FROM__START
+FROM koalaman/shellcheck:stable as shellcheck
 FROM mvdan/shfmt:latest-alpine as shfmt
 FROM hadolint/hadolint:v2.12.0-alpine as hadolint
+FROM mrtazz/checkmake:latest as checkmake
 FROM zricethezav/gitleaks:v8.15.3 as gitleaks
 #FROM__END
 
@@ -165,38 +168,34 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 #############################################################################################
 
 #COPY__START
-COPY --from=shfmt /bin/shfmt /usr/bin/
-COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
-COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
+COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
+COPY --link --from=shfmt /bin/shfmt /usr/bin/
+COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
+COPY --link --from=checkmake /checkmake /usr/bin/checkmake
+COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
 #COPY__END
 
 #############################################################################################
 ## @generated by .automation/build.py using descriptor files, please do not update manually ##
 #############################################################################################
 #OTHER__START
 # shellcheck installation
-RUN ML_THIRD_PARTY_DIR="/third-party/shellcheck" \
-    && mkdir -p ${ML_THIRD_PARTY_DIR} \
-    && wget -qO- "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv --directory ${ML_THIRD_PARTY_DIR} \
-    && mv "${ML_THIRD_PARTY_DIR}/shellcheck-stable/shellcheck" /usr/bin/ \
-    && find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \
+# Managed with COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
 
 # shfmt installation
-# Managed with COPY --from=shfmt /bin/shfmt /usr/bin/
+# Managed with COPY --link --from=shfmt /bin/shfmt /usr/bin/
 
 # hadolint installation
-# Managed with COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
+# Managed with COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
 
 # dotenv-linter installation
-    && wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \
+RUN wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \
 
 # checkmake installation
-    && ( [ -d /usr/local/bin ] || mkdir -p /usr/local/bin ) \
-    && wget -q "https://github.com/mrtazz/checkmake/releases/download/0.2.1/checkmake-0.2.1.linux.amd64" -O /usr/local/bin/checkmake \
-    && chmod 755 /usr/local/bin/checkmake \
+# Managed with COPY --link --from=checkmake /checkmake /usr/bin/checkmake
 
 # gitleaks installation
-# Managed with COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
+# Managed with COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
 
 # trivy installation
     && wget --tries=5 -q -O - https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin