Skip to content

Commit d750b24

Browse files
nicholasdillenicholasdille
committed
Added download cache
1 parent 706cc82 commit d750b24

File tree

813 files changed

+7253
-3567
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

813 files changed

+7253
-3567
lines changed

‎.github/workflows/check.yml

+42
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ jobs:
2424
tools: ${{ steps.collect.outputs.tools }}
2525
steps:
2626

27+
- name: Checkout code
28+
uses: actions/checkout@v4
29+
with:
30+
fetch-depth: 0
31+
2732
- name: Install uniget
2833
uses: uniget-org/uniget-action@v1
2934
with:
@@ -56,7 +61,44 @@ jobs:
5661
| helper/usr/local/bin/jq --slurp --compact-output '.'
5762
)"
5863
echo " JSON: ${TOOLS_JSON}"
64+
65+
TOOLS_JSON_LEN="$(
66+
echo -n "${TOOLS}" \
67+
| jq 'length'
68+
)"
69+
echo " TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
70+
71+
if test "${TOOLS_JSON_LEN}" -ge 256; then
72+
echo "### Too many tools to process: ${TOOLS_JSON_LEN}."
73+
echo "### Fallback to last commit"
74+
75+
TOOLS="$(
76+
git log -1 --pretty=format: --name-only --diff-filter=d \
77+
| grep ^tools/ \
78+
| cut -d/ -f2 \
79+
| sort \
80+
| uniq \
81+
| xargs echo
82+
)"
83+
echo "### Got tools=${TOOLS}."
84+
85+
TOOLS_JSON="$(
86+
echo -n "${TOOLS}" \
87+
| tr ' ' '\n' \
88+
| helper/usr/local/bin/jq --raw-input '.' \
89+
| helper/usr/local/bin/jq --slurp --compact-output '.'
90+
)"
91+
echo " JSON: ${TOOLS_JSON}"
92+
93+
TOOLS_JSON_LEN="$(
94+
echo -n "${TOOLS}" \
95+
| jq 'length'
96+
)"
97+
echo " TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
98+
fi
99+
59100
echo "tools=${TOOLS_JSON}" >>"${GITHUB_OUTPUT}"
101+
echo "tools_len=${TOOLS_JSON_LEN}" >>"${GITHUB_OUTPUT}"
60102
61103
build:
62104
name: Build container image

‎.github/workflows/tag.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
CHANGED_TOOLS="$(
6767
git log --pretty=format: --name-only --diff-filter=d ${OLD_COMMIT_SHA}..${GITHUB_SHA} \
6868
| sort \
69-
| grep -E "^tools/[^/]+/" \
69+
| grep "^tools/" \
7070
| cut -d/ -f2 \
7171
| uniq \
7272
| while read -r TOOL; do test -d "tools/${TOOL}" && echo "${TOOL}"; done \

‎@template/Dockerfile.template

+58-30
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ ARG name
1818
ARG version
1919

2020
# tarball
21-
#RUN <<EOF
21+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
2222
#echo "### Setting architecture string for ${TARGETPLATFORM}"
2323
#case "${arch}" in
2424
# x86_64)
@@ -34,14 +34,20 @@ ARG version
3434
#esac
3535
#echo " Using ${arch_suffix}"
3636
#
37-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
38-
#check-download ""
39-
#curl --silent --show-error --location --fail "" \
40-
#| tar --extract --gzip --directory="${prefix}/bin/" --no-same-owner
37+
#url=""
38+
#filename="$( basename "${url}" )"
39+
#
40+
#check-download "${url}"
41+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
42+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
43+
# "${url}"
44+
#
45+
#tar --file="${uniget_cache_download}/${filename}" --list
46+
#tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --no-same-owner
4147
#EOF
4248

4349
# executable
44-
#RUN <<EOF
50+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
4551
#echo "### Setting architecture string for ${TARGETPLATFORM}"
4652
#case "${arch}" in
4753
# x86_64)
@@ -57,15 +63,21 @@ ARG version
5763
#esac
5864
#echo " Using ${arch_suffix}"
5965
#
60-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
61-
#check-download ""
62-
#curl --silent --show-error --location --fail --output "${prefix}/bin/foo" \
63-
# ""
64-
#chmod +x "${prefix}/bin/foo"
66+
#url=""
67+
#filename="$( basename "${url}" )"
68+
#
69+
#check-download "${url}"
70+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
71+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
72+
# "${url}"
73+
#
74+
#install --mode=0755 \
75+
# "${uniget_cache_download}/${filename}" \
76+
# "${prefix}/bin/foo"
6577
#EOF
6678

6779
# zip
68-
#RUN <<EOF
80+
#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
6981
#echo "### Setting architecture string for ${TARGETPLATFORM}"
7082
#case "${arch}" in
7183
# x86_64)
@@ -81,19 +93,33 @@ ARG version
8193
#esac
8294
#echo " Using ${arch_suffix}"
8395
#
84-
#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
85-
#check-download ""
8696
#url=""
8797
#filename="$(basename "${url}")"
88-
#curl --silent --show-error --location --fail --remote-name "${url}"
89-
#unzip -q -o -d "${prefix}/bin" "${filename}"
98+
#
99+
#check-download "${url}"
100+
#check-github-release-asset "foo/bar" "v${version}" "${filename}"
101+
#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
102+
# "${url}"
103+
#
104+
#unzip -q -o -d "${prefix}/bin" "${uniget_cache_download}/${filename}"
90105
#EOF
91106

92107
# go
108+
#RUN --mount=type=cache,target=/var/cache/apt <<EOF
109+
#apt-get update
110+
#apt-get -y install --no-install-recommends \
111+
# foo
112+
#EOF
113+
#RUN --mount=type=cache,target=/var/cache/apk <<EOF
114+
#apk add --update-cache \
115+
# foo
116+
#EOF
93117
#COPY --link --from=go / /usr/local/
94118
#COPY --link --from=make / /usr/local/
95119
#WORKDIR /go/src/github.com/foo/bar
96-
#RUN <<EOF
120+
#ARG name
121+
#ARG version
122+
#RUN --mount=type=cache,target=/root/go/pkg/mod <<EOF
97123
#check-clone "https://github.com/foo/bar" "v${version}"
98124
#git clone -q --config advice.detachedHead=false --depth 1 --branch "v${version}" https://github.com/foo/bar .
99125
#make
@@ -103,7 +129,7 @@ ARG version
103129
# rust
104130
#COPY --link --from=rust / /usr/local/
105131
#WORKDIR /tmp/github.com/foo/bar
106-
#RUN <<EOF
132+
#RUN <<EOF # no download cache
107133
#git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
108134
#export CARGO_HOME=/usr/local/cargo
109135
#export RUSTUP_HOME=/usr/local/rustup
@@ -115,14 +141,7 @@ ARG version
115141
# python
116142
#COPY --link --from=python / /usr/local/
117143
#COPY --link --from=shiv / /usr/local/
118-
#RUN <<EOF
119-
#apt-get update
120-
#apt-get -y install --no-install-recommends \
121-
# python3-pip \
122-
# python3-venv \
123-
# python-is-python3
124-
#EOF
125-
#RUN <<EOF
144+
#RUN --mount=type=cache,target=/root/.cache/pip <<EOF # no download cache
126145
#shiv --output-file "${prefix}/bin/foo" --console-script foo "foo==${version}"
127146
#EOF
128147

@@ -132,30 +151,39 @@ ARG version
132151
#WORKDIR /uniget_bootstrap/libexec/foo
133152
#ARG name
134153
#ARG version
135-
#RUN <<EOF
154+
#RUN <<EOF # no download cache
136155
#npm install \
137156
# --omit=dev \
138157
# "foo@${version}"
139158
#ln --symbolic --relative --force "${prefix}/libexec/foo/node_modules/.bin/foo" "${prefix}/bin/"
140159
#EOF
141160

142161
# c
162+
#RUN --mount=type=cache,target=/var/cache/apt <<EOF
163+
#apt-get update
164+
#apt-get -y install --no-install-recommends \
165+
# foo
166+
#EOF
167+
#RUN --mount=type=cache,target=/var/cache/apk <<EOF
168+
#apk add --update-cache \
169+
# foo
170+
#EOF
143171
#COPY --link --from=make / /usr/local/
144172
#WORKDIR /tmp/foo
145-
#RUN <<EOF
173+
#RUN <<EOF # no download cache
146174
#git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
147175
#./configure --prefix="${prefix}"
148176
#make LDFLAGS=-static
149177
#make install
150178
#EOF
151179

152180
# dummy for testing
153-
#RUN <<EOF
181+
#RUN <<EOF # no download cache
154182
#touch "${prefix}/foo"
155183
#EOF
156184

157185
# completion
158-
#RUN <<EOF
186+
#RUN <<EOF # no download cache
159187
#"${prefix}/bin/foo" completion bash >"${prefix}/share/bash-completion/completions/foo"
160188
#"${prefix}/bin/foo" completion fish >"${prefix}/share/fish/vendor_completions.d/foo.fish"
161189
#"${prefix}/bin/foo" completion zsh >"${prefix}/share/zsh/vendor-completions/_foo"

‎renovate.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@
354354
{
355355
"customType": "regex",
356356
"datasourceTemplate": "github-releases",
357-
"depNameTemplate": "foo/bar",
357+
"depNameTemplate": "warpstreamlabs/bento",
358358
"extractVersionTemplate": "^v(?<version>.+)$",
359359
"fileMatch": [
360360
"^tools/bento/manifest.yaml$"

‎scripts/rebuild-all.sh

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
set -o errexit
3+
4+
if test -z "${GITHUB_TOKEN}"; then
5+
echo "ERROR: GITHUB_TOKEN is not set"
6+
exit 1
7+
fi
8+
: "${ref:=main}"
9+
: "${workflow:=rebuild.yml}"
10+
: "${repository:=uniget-org/tools}"
11+
: "${batch_size:=100}"
12+
13+
function dispatch() {
14+
local repository="${1}"
15+
local ref="${2}"
16+
local workflow="${3}"
17+
local tools="${4}"
18+
local dryrun="${5:-false}"
19+
20+
CURL="curl"
21+
if test -n "${dryrun}" && test "${dryrun}" == "true"; then
22+
CURL="echo curl"
23+
fi
24+
25+
${CURL} --silent --show-error --location --fail \
26+
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
27+
--url "https://api.github.com/repos/${repository}/actions/workflows/${workflow}/dispatches" \
28+
--data "{\"ref\": \"${ref}\", \"inputs\": {\"tool\": \"$(echo "${tools}" | xargs echo)\"}}"
29+
}
30+
31+
make metadata.json
32+
33+
TOOLS_DEPS="$(
34+
jq --raw-output '.tools[] | select(.build_dependencies != null) | .build_dependencies[]' <metadata.json \
35+
| sort \
36+
| uniq
37+
)"
38+
dispatch "${repository}" "${ref}" "${workflow}" "${TOOLS_DEPS}"
39+
TOOLS_DEPS_REGEX_OR="$(
40+
echo "${TOOLS_DEPS}" \
41+
| jq --raw-input --raw-output '[inputs] | join("|")'
42+
)"
43+
44+
TOOLS="$(
45+
jq --raw-output '.tools[].name' <metadata.json \
46+
| sort \
47+
| grep -E -v "(${TOOLS_DEPS_REGEX_OR})"
48+
)"
49+
SHUFFLED_TOOLS="$(
50+
echo "${TOOLS}" | shuf
51+
)"
52+
TOOLS_COUNT="$(echo "${TOOLS}" | wc -l)"
53+
54+
start_index=1
55+
end_index="${TOOLS_COUNT}"
56+
echo "### Processing ${TOOLS_COUNT} tools"
57+
while test "${start_index}" -lt "${end_index}"; do
58+
echo "### Starting at ${start_index}"
59+
NEXT_TOOLS="$(
60+
echo "${SHUFFLED_TOOLS}" | tail -n +$(( start_index + 1 )) | head -n "${batch_size}"
61+
)"
62+
63+
dispatch "${repository}" "${ref}" "${workflow}" "${NEXT_TOOLS}" true
64+
65+
start_index=$(( start_index + 100 ))
66+
done

‎scripts/rewrite-add-download-cache.sh

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
set -o errexit
3+
4+
find tools -type f -name Dockerfile.template \
5+
| sort \
6+
| while read -r FILENAME; do
7+
8+
if grep -P -z -q 'RUN <<EOF # no download cache\n' "${FILENAME}"; then
9+
#echo "Ignoring ${FILENAME}"
10+
continue
11+
fi
12+
13+
if grep -P -z -q 'RUN <<EOF\nnpm install ' "${FILENAME}"; then
14+
#echo "Ignoring ${FILENAME}"
15+
continue
16+
fi
17+
18+
if grep -P -z -q 'RUN <<EOF\nshiv ' "${FILENAME}"; then
19+
#echo "Ignoring ${FILENAME}"
20+
continue
21+
fi
22+
23+
if grep -q "^unzip -q " "${FILENAME}"; then
24+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?url="([^"]+)"\nfilename=".+"\ncurl --silent --show-error --location --fail --remote-name "([^"]+)"\nunzip -q -o -d "([^"]+)" "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\nunzip -t "${uniget_cache_download}/${filename}"\nunzip -q -o -d "\6" "${uniget_cache_download}/${filename}"@' "${FILENAME}"
25+
continue
26+
fi
27+
28+
if grep -P -z -q 'RUN <<EOF\ncurl ' "${FILENAME}"; then
29+
echo "Missing handler ${FILENAME}"
30+
continue
31+
fi
32+
33+
if grep -P -z -q 'RUN <<EOF\ngit clone ' "${FILENAME}"; then
34+
#sed -E -z -i 's@RUN <<EOF\ngit clone -q --config advice.detachedHead=false --depth 1 --branch "([^"]+)" https://github.com/([^ ]+) .@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="https://github.com/\2/archive/refs/tags/\1.tar.gz"\nfilename="$( basename "${url}" )"\n\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n@' "${FILENAME}"
35+
#echo "Ignoring ${FILENAME}"
36+
continue
37+
fi
38+
39+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail "[^"]+" \\\n| tar' "${FILENAME}"; then
40+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @' "${FILENAME}"
41+
continue
42+
fi
43+
44+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
45+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\6"\n@' "${FILENAME}"
46+
continue
47+
fi
48+
49+
if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n' "${FILENAME}"; then
50+
sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\ncheck-github-release-asset "\1" "\2" "\3"\n@' "${FILENAME}"
51+
continue
52+
fi
53+
54+
if grep -P -z -q 'RUN <<EOF\ncheck-download "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
55+
sed -E -z -i 's@RUN <<EOF\ncheck-download "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\1"\nfilename="$( basename "${url}" )"\n\ncheck-download "${url}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\4"\n@' "${FILENAME}"
56+
continue
57+
fi
58+
59+
if grep -P -z -q 'RUN <<EOF\necho "### Setting' "${FILENAME}" && grep -P -z -q 'check-github-release-asset' "${FILENAME}"; then
60+
sed -E -z -i 's@echo "### Downloading [^"]+"\n@@; s@RUN <<EOF\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\n@; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @url="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@url="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n "${url}"\n\ninstall --mode=0755 \\\n "${uniget_cache_download}/${filename}" \\\n "\6"\n@' "${FILENAME}"
61+
continue
62+
fi
63+
64+
if grep -q "RUN <<EOF" "${FILENAME}"; then
65+
echo "Unhandled ${FILENAME}"
66+
fi
67+
68+
done

0 commit comments

Comments
 (0)