Skip to content

Commit cfc074d

Browse files
nicholasdillenicholasdille
committed
Added download cache
1 parent ee60baf commit cfc074d

File tree

808 files changed

+7163
-3558
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

808 files changed

+7163
-3558
lines changed

‎.github/workflows/check.yml

+42
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ jobs:
2424
tools: ${{ steps.collect.outputs.tools }}
2525
steps:
2626

27+
- name: Checkout code
28+
uses: actions/checkout@v4
29+
with:
30+
fetch-depth: 0
31+
2732
- name: Install uniget
2833
uses: uniget-org/uniget-action@v1
2934
with:
@@ -56,7 +61,44 @@ jobs:
5661
| helper/usr/local/bin/jq --slurp --compact-output '.'
5762
)"
5863
echo " JSON: ${TOOLS_JSON}"
64+
65+
TOOLS_JSON_LEN="$(
66+
echo -n "${TOOLS}" \
67+
| jq 'length'
68+
)"
69+
echo " TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
70+
71+
if test "${TOOLS_JSON_LEN}" -ge 256; then
72+
echo "### Too many tools to process: ${TOOLS_JSON_LEN}."
73+
echo "### Fallback to last commit"
74+
75+
TOOLS="$(
76+
git log -1 --pretty=format: --name-only --diff-filter=d \
77+
| grep ^tools/ \
78+
| cut -d/ -f2 \
79+
| sort \
80+
| uniq \
81+
| xargs echo
82+
)"
83+
echo "### Got tools=${TOOLS}."
84+
85+
TOOLS_JSON="$(
86+
echo -n "${TOOLS}" \
87+
| tr ' ' '\n' \
88+
| helper/usr/local/bin/jq --raw-input '.' \
89+
| helper/usr/local/bin/jq --slurp --compact-output '.'
90+
)"
91+
echo " JSON: ${TOOLS_JSON}"
92+
93+
TOOLS_JSON_LEN="$(
94+
echo -n "${TOOLS}" \
95+
| jq 'length'
96+
)"
97+
echo " TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
98+
fi
99+
59100
echo "tools=${TOOLS_JSON}" >>"${GITHUB_OUTPUT}"
101+
echo "tools_len=${TOOLS_JSON_LEN}" >>"${GITHUB_OUTPUT}"
60102
61103
build:
62104
name: Build container image

‎.github/workflows/tag.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
CHANGED_TOOLS="$(
6767
git log --pretty=format: --name-only --diff-filter=d ${OLD_COMMIT_SHA}..${GITHUB_SHA} \
6868
| sort \
69-
| grep -E "^tools/[^/]+/" \
69+
| grep "^tools/" \
7070
| cut -d/ -f2 \
7171
| uniq \
7272
| while read -r TOOL; do test -d "tools/${TOOL}" && echo "${TOOL}"; done \

‎@template/Dockerfile.template

+58-30
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ ARG name
1818
ARG version
1919

2020
# tarball
21-
#RUN <<EOF
21+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
2222
#echo "### Setting architecture string for ${TARGETPLATFORM}"
2323
#case "${arch}" in
2424
# x86_64)
@@ -34,14 +34,20 @@ ARG version
3434
#esac
3535
#echo " Using ${arch_suffix}"
3636
#
37-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
38-
#check-download ""
39-
#curl --silent --show-error --location --fail "" \
40-
#| tar --extract --gzip --directory="${prefix}/bin/" --no-same-owner
37+
#url=""
38+
#filename="$( basename "${url}" )"
39+
#
40+
#check-download "${url}"
41+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
42+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
43+
# "${url}"
44+
#
45+
#tar --file="${uniget_cache_download}/${filename}" --list
46+
#tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --no-same-owner
4147
#EOF
4248

4349
# executable
44-
#RUN <<EOF
50+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
4551
#echo "### Setting architecture string for ${TARGETPLATFORM}"
4652
#case "${arch}" in
4753
# x86_64)
@@ -57,15 +63,21 @@ ARG version
5763
#esac
5864
#echo " Using ${arch_suffix}"
5965
#
60-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
61-
#check-download ""
62-
#curl --silent --show-error --location --fail --output "${prefix}/bin/foo" \
63-
# ""
64-
#chmod +x "${prefix}/bin/foo"
66+
#url=""
67+
#filename="$( basename "${url}" )"
68+
#
69+
#check-download "${url}"
70+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
71+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
72+
# "${url}"
73+
#
74+
#install --mode=0755 \
75+
# "${uniget_cache_download}/${filename}" \
76+
# "${prefix}/bin/foo"
6577
#EOF
6678

6779
# zip
68-
#RUN <<EOF
80+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
6981
#echo "### Setting architecture string for ${TARGETPLATFORM}"
7082
#case "${arch}" in
7183
# x86_64)
@@ -81,19 +93,33 @@ ARG version
8193
#esac
8294
#echo " Using ${arch_suffix}"
8395
#
84-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
85-
#check-download ""
8696
#url=""
8797
#filename="$(basename "${url}")"
88-
#curl --silent --show-error --location --fail --remote-name "${url}"
89-
#unzip -q -o -d "${prefix}/bin" "${filename}"
98+
#
99+
#check-download "${url}"
100+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
101+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
102+
# "${url}"
103+
#
104+
#unzip -q -o -d "${prefix}/bin" "${uniget_cache_download}/${filename}"
90105
#EOF
91106

92107
# go
108+
#RUN --mount=type=cache,target=/var/cache/apt <<EOF
109+
#apt-get update
110+
#apt-get -y install --no-install-recommends \
111+
# foo
112+
#EOF
113+
#RUN --mount=type=cache,target=/var/cache/apk <<EOF
114+
#apk add --update-cache \
115+
# foo
116+
#EOF
93117
#COPY --link --from=go / /usr/local/
94118
#COPY --link --from=make / /usr/local/
95119
#WORKDIR /go/src/github.com/foo/bar
96-
#RUN <<EOF
120+
#ARG name
121+
#ARG version
122+
#RUN --mount=type=cache,target=/root/go/pkg/mod <<EOF
97123
#check-clone "https://github.com/foo/bar" "v${version}"
98124
#git clone -q --config advice.detachedHead=false --depth 1 --branch "v${version}" https://github.com/foo/bar .
99125
#make
@@ -103,7 +129,7 @@ ARG version
103129
# rust
104130
#COPY --link --from=rust / /usr/local/
105131
#WORKDIR /tmp/github.com/foo/bar
106-
#RUN <<EOF
132+
#RUN <<EOF # no download cache
107133
#git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
108134
#export CARGO_HOME=/usr/local/cargo
109135
#export RUSTUP_HOME=/usr/local/rustup
@@ -115,14 +141,7 @@ ARG version
115141
# python
116142
#COPY --link --from=python / /usr/local/
117143
#COPY --link --from=shiv / /usr/local/
118-
#RUN <<EOF
119-
#apt-get update
120-
#apt-get -y install --no-install-recommends \
121-
# python3-pip \
122-
# python3-venv \
123-
# python-is-python3
124-
#EOF
125-
#RUN <<EOF
144+
#RUN --mount=type=cache,target=/root/.cache/pip <<EOF # no download cache
126145
#shiv --output-file "${prefix}/bin/foo" --console-script foo "foo==${version}"
127146
#EOF
128147

@@ -132,30 +151,39 @@ ARG version
132151
#WORKDIR /uniget_bootstrap/libexec/foo
133152
#ARG name
134153
#ARG version
135-
#RUN <<EOF
154+
#RUN <<EOF # no download cache
136155
#npm install \
137156
# --omit=dev \
138157
# "foo@${version}"
139158
#ln --symbolic --relative --force "${prefix}/libexec/foo/node_modules/.bin/foo" "${prefix}/bin/"
140159
#EOF
141160

142161
# c
162+
#RUN --mount=type=cache,target=/var/cache/apt <<EOF
163+
#apt-get update
164+
#apt-get -y install --no-install-recommends \
165+
# foo
166+
#EOF
167+
#RUN --mount=type=cache,target=/var/cache/apk <<EOF
168+
#apk add --update-cache \
169+
# foo
170+
#EOF
143171
#COPY --link --from=make / /usr/local/
144172
#WORKDIR /tmp/foo
145-
#RUN <<EOF
173+
#RUN <<EOF # no download cache
146174
#git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
147175
#./configure --prefix="${prefix}"
148176
#make LDFLAGS=-static
149177
#make install
150178
#EOF
151179

152180
# dummy for testing
153-
#RUN <<EOF
181+
#RUN <<EOF # no download cache
154182
#touch "${prefix}/foo"
155183
#EOF
156184

157185
# completion
158-
#RUN <<EOF
186+
#RUN <<EOF # no download cache
159187
#"${prefix}/bin/foo" completion bash >"${prefix}/share/bash-completion/completions/foo"
160188
#"${prefix}/bin/foo" completion fish >"${prefix}/share/fish/vendor_completions.d/foo.fish"
161189
#"${prefix}/bin/foo" completion zsh >"${prefix}/share/zsh/vendor-completions/_foo"

‎scripts/rewrite-add-download-cache.sh

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
set -o errexit
3+
4+
find tools -type f -name Dockerfile.template \
5+
| sort \
6+
| while read -r FILENAME; do
7+
8+
if grep -P -z -q 'RUN <<EOF # no download cache\n' "${FILENAME}"; then
9+
#echo "Ignoring ${FILENAME}"
10+
continue
11+
fi
12+
13+
if grep -P -z -q 'RUN <<EOF\nnpm install ' "${FILENAME}"; then
14+
#echo "Ignoring ${FILENAME}"
15+
continue
16+
fi
17+
18+
if grep -P -z -q 'RUN <<EOF\nshiv ' "${FILENAME}"; then
19+
#echo "Ignoring ${FILENAME}"
20+
continue
21+
fi
22+
23+
if grep -q "^unzip -q " "${FILENAME}"; then
24+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?url="([^"]+)"\nfilename=".+"\ncurl --silent --show-error --location --fail --remote-name "([^"]+)"\nunzip -q -o -d "([^"]+)" "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\nunzip -t "${uniget_cache_download}/${filename}"\nunzip -q -o -d "\6" "${uniget_cache_download}/${filename}"@' "${FILENAME}"
25+
continue
26+
fi
27+
28+
if grep -P -z -q 'RUN <<EOF\ncurl ' "${FILENAME}"; then
29+
echo "Missing handler ${FILENAME}"
30+
continue
31+
fi
32+
33+
if grep -P -z -q 'RUN <<EOF\ngit clone ' "${FILENAME}"; then
34+
#sed -E -z -i 's@RUN <<EOF\ngit clone -q --config advice.detachedHead=false --depth 1 --branch "([^"]+)" https://github.com/([^ ]+) .@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="https://github.com/\2/archive/refs/tags/\1.tar.gz"\nfilename="$( basename "${url}" )"\n\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n@' "${FILENAME}"
35+
#echo "Ignoring ${FILENAME}"
36+
continue
37+
fi
38+
39+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail "[^"]+" \\\n| tar' "${FILENAME}"; then
40+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @' "${FILENAME}"
41+
continue
42+
fi
43+
44+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
45+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\6"\n@' "${FILENAME}"
46+
continue
47+
fi
48+
49+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n' "${FILENAME}"; then
50+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\ncheck-github-release-asset "\1" "\2" "\3"\n@' "${FILENAME}"
51+
continue
52+
fi
53+
54+
if grep -P -z -q 'RUN <<EOF\ncheck-download "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
55+
sed -E -z -i 's@RUN <<EOF\ncheck-download "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\1"\nfilename="$( basename "${url}" )"\n\ncheck-download "${url}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\4"\n@' "${FILENAME}"
56+
continue
57+
fi
58+
59+
if grep -P -z -q 'RUN <<EOF\necho "### Setting' "${FILENAME}" && grep -P -z -q 'check-github-release-asset' "${FILENAME}"; then
60+
sed -E -z -i 's@echo "### Downloading [^"]+"\n@@; s@RUN <<EOF\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\n@; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @url="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@url="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\6"\n@' "${FILENAME}"
61+
continue
62+
fi
63+
64+
if grep -q "RUN <<EOF" "${FILENAME}"; then
65+
echo "Unhandled ${FILENAME}"
66+
fi
67+
68+
done

‎tools/acorn/Dockerfile.template

+9-4
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,16 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
77
SHELL [ "bash", "-clo", "errexit" ]
88
ARG name
99
ARG version
10-
RUN <<EOF
11-
check-github-release-asset "acorn-io/runtime" "v${version}" "acorn-v${version}-linux-${alt_arch}.tar.gz"
10+
RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
11+
url="https://github.com/acorn-io/runtime/releases/download/v${version}/acorn-v${version}-linux-${alt_arch}.tar.gz"
12+
filename="$( basename "${url}" )"
1213

13-
curl --silent --show-error --location --fail "https://github.com/acorn-io/runtime/releases/download/v${version}/acorn-v${version}-linux-${alt_arch}.tar.gz" \
14-
| tar --extract --gzip --directory="${prefix}/bin/" --no-same-owner \
14+
check-github-release-asset "acorn-io/runtime" "v${version}" "${filename}"
15+
curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
16+
"${url}"
17+
18+
tar --file="${uniget_cache_download}/${filename}" --list
19+
tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin/" --no-same-owner \
1520
acorn
1621

1722
"${prefix}/bin/acorn" completion bash >"${prefix}/share/bash-completion/completions/acorn"

‎tools/acr/Dockerfile.template

+3-3
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
88
/etc/profile.d/ \
99
/etc/profile.d/
1010
SHELL [ "bash", "-clo", "errexit" ]
11-
ARG name
12-
ARG version
1311
COPY --link --from=go / /usr/local/
1412
COPY --link --from=make / /usr/local/
1513
WORKDIR /go/src/github.com/Azure/acr-cli
16-
RUN <<EOF
14+
ARG name
15+
ARG version
16+
RUN <<EOF # no download cache
1717
git clone -q --config advice.detachedHead=false --depth 1 --branch "v${version}" https://github.com/Azure/acr-cli .
1818
make -j$(nproc) binaries GO_LDFLAGS="-ldflags '-s -w -linkmode external -extldflags -static'"
1919
cp bin/acr "${prefix}/bin/"

‎tools/act/Dockerfile.template

+10-4
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,15 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
77
SHELL [ "bash", "-clo", "errexit" ]
88
ARG name
99
ARG version
10-
RUN <<EOF
11-
check-github-release-asset "nektos/act" "v${version}" "act_Linux_x86_64.tar.gz"
12-
curl --silent --show-error --location --fail "https://github.com/nektos/act/releases/download/v${version}/act_Linux_${arch}.tar.gz" \
13-
| tar --extract --gzip --directory="${prefix}/bin" --no-same-owner \
10+
RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
11+
url="https://github.com/nektos/act/releases/download/v${version}/act_Linux_${arch}.tar.gz"
12+
filename="$( basename "${url}" )"
13+
14+
check-github-release-asset "nektos/act" "v${version}" "${filename}"
15+
curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
16+
"${url}"
17+
18+
tar --file="${uniget_cache_download}/${filename}" --list
19+
tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --no-same-owner \
1420
act
1521
EOF

‎tools/actions-runner/Dockerfile.template

+9-5
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ SHELL [ "bash", "-clo", "errexit" ]
88
WORKDIR /uniget_bootstrap/opt/actions-runner
99
ARG name
1010
ARG version
11-
RUN <<EOF
11+
RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
1212
echo "### Setting architecture string for ${TARGETPLATFORM}"
1313
case "${arch}" in
1414
x86_64)
@@ -24,9 +24,13 @@ case "${arch}" in
2424
esac
2525
echo " Using ${arch_suffix}"
2626

27-
check-github-release-asset "actions/runner" "v${version}" "actions-runner-linux-${arch_suffix}-${version}.tar.gz"
27+
url="https://github.com/actions/runner/releases/download/v${version}/actions-runner-linux-${arch_suffix}-${version}.tar.gz"
28+
filename="$( basename "${url}" )"
2829

29-
echo "### Downloading actions-runner ${version}"
30-
curl --silent --show-error --location --fail "https://github.com/actions/runner/releases/download/v${version}/actions-runner-linux-${arch_suffix}-${version}.tar.gz" \
31-
| tar --extract --gzip --no-same-owner
30+
check-github-release-asset "actions/runner" "v${version}" "${filename}"
31+
curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
32+
"${url}"
33+
34+
tar --file="${uniget_cache_download}/${filename}" --list
35+
tar --file="${uniget_cache_download}/${filename}" --extract --gzip --no-same-owner
3236
EOF

0 commit comments

Comments
 (0)