diff --git a/.ci/actions.py b/.ci/actions.py old mode 100644 new mode 100755 index 15b8137..649c641 --- a/.ci/actions.py +++ b/.ci/actions.py @@ -22,8 +22,10 @@ def main(): # render templates render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-cuda.yml" , platform="cuda", type="normal") render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-cuda-empty.yml", platform="cuda", type="empty") - render_template(f"{i}/sh-build-iso.yml.jinja2", f"{o}/sh-build-iso-rocm.yml" , platform="rocm", type="normal") + render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-cuda-gui.yml" , platform="cuda", type="gui") + render_template(f"{i}/sh-build-iso.yml.jinja2", f"{o}/sh-build-iso-rocm.yml" , platform="rocm", type="normal", delay=0) render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-rocm-empty.yml", platform="rocm", type="empty") + render_template(f"{i}/sh-build-iso.yml.jinja2", f"{o}/sh-build-iso-rocm-gui.yml" , platform="rocm", type="gui", delay=30) if __name__ == "__main__": main() diff --git a/.ci/configure.py b/.ci/configure.py index 8b2dba7..408aa09 100755 --- a/.ci/configure.py +++ b/.ci/configure.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import configparser import glob import jinja2 import sys @@ -24,6 +25,9 @@ def main(): # and normal iso type = "normal" + # and empty options + options = {} + # set platform if len(sys.argv) >= 2: platform = sys.argv[1] @@ -32,6 +36,16 @@ def main(): if len(sys.argv) >= 3: type = sys.argv[2] + # load options + config = configparser.ConfigParser() + config.read(".ci/options.ini") + + # repositories + options["repositories"] = dict(config.items("repositories")) + + # revisions + options["revisions"] = dict(config.items("revisions")) + # list of rendered files rendered = [] @@ -41,7 +55,7 @@ def main(): rendered.sort() # render file - render_template(filepath, platform=platform, type=type, rendered=rendered) + render_template(filepath, platform=platform, type=type, rendered=rendered, **options) # add output file to rendered list rendered.append(filepath[:-7].replace("\\", "/")) diff --git a/.ci/options.ini b/.ci/options.ini new file mode 100644 index 0000000..f3add5d --- /dev/null +++ b/.ci/options.ini @@ -0,0 +1,22 @@ +[repositories] +automatic = https://github.com/vladmandic/automatic.git +axolotl = https://github.com/OpenAccess-AI-Collective/axolotl.git +comfyui = https://github.com/comfyanonymous/ComfyUI.git +koboldcpp = https://github.com/LostRuins/koboldcpp.git +llamacpp = https://github.com/ggerganov/llama.cpp.git +sillytavern = https://github.com/SillyTavern/SillyTavern.git +sillytavern_extras = https://github.com/SillyTavern/SillyTavern-Extras.git +text_generation_webui = https://github.com/oobabooga/text-generation-webui.git +vllm = https://github.com/vllm-project/vllm.git + +[revisions] +automatic = de267390905335d353deab3e9ac2031432f4d305 +axolotl = 68601ec6ad1cc0e8cb855376586e6eef6a8aa270 +comfyui = 8dc19e40d129c8ee049be7be2657458509717ba5 +koboldcpp = 593f08bb78ab6035f91cc7aa753975254f30ed82 +llamacpp = 784e11dea1f5ce9638851b2b0dddb107e2a609c8 +sillytavern = 47b656260595967ad70b749845f7b5b33ed3631c +sillytavern_extras = 1d82f3a8607319d1e09a2f4749a09c564c18c320 +text_generation_webui = ad122361ea1dbc4b419214891b83f6b9ac12f947 +vllm = a395a638c2f18d549e7d01655cf7a6dbee566f91 + diff --git a/.ci/template/gh-build-iso.yml.jinja2 b/.ci/template/gh-build-iso.yml.jinja2 index 2d8e583..cfcaf10 100644 --- a/.ci/template/gh-build-iso.yml.jinja2 +++ b/.ci/template/gh-build-iso.yml.jinja2 @@ -23,10 +23,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 -{% if type != "empty" %} - with: - submodules: recursive -{% endif %} - name: Build image uses: addnab/docker-run-action@v3 diff --git a/.ci/template/sh-build-iso.yml.jinja2 b/.ci/template/sh-build-iso.yml.jinja2 index 3e8921c..7508eb1 100644 --- a/.ci/template/sh-build-iso.yml.jinja2 +++ b/.ci/template/sh-build-iso.yml.jinja2 @@ -5,10 +5,6 @@ on: branches: - main - pull_request: - branches: - - main - jobs: vm-start: if: always() @@ -34,6 +30,33 @@ jobs: # Fix permissions for private key chmod 600 /tmp/private.key + - name: Wait {{ delay }} seconds + run: | + sleep {{ delay }} + + - name: Wait until Github Actions Runner VM is stopped + run: | + ok="0" + + while true; do + status=$(ssh runner "virsh domstate GithubActions") + + if [ "$status" = "shutoff" ]; then + if [ "$ok" = "1" ]; then + echo "VM is shut off. Exiting." + break + else + sleep $((10 + $RANDOM % 10)) + ok="1" + fi + else + ok="0" + fi + + echo "VM is not shut off yet. Waiting for 5 seconds..." + sleep 5 + done + - name: Start Github Actions Runner VM run: | ssh runner "virsh start GithubActions" @@ -49,10 +72,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 -{% if type != "empty" %} - with: - submodules: recursive -{% endif %} - name: Build image uses: addnab/docker-run-action@v3 @@ -131,6 +150,3 @@ jobs: # Wait 2 minutes sleep 120 - - # Force terminate if needed - ssh runner "virsh destroy GithubActions" || true diff --git a/.ci/update.py b/.ci/update.py new file mode 100755 index 0000000..f03d462 --- /dev/null +++ b/.ci/update.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import configparser +import requests + +def fetch_latest_revision(url): + # github + if url.startswith("https://github.com/") and url.endswith(".git"): + # extract repo path + path = url[19:-4] + + # get commits from api + response = requests.get(f"https://api.github.com/repos/{path}/commits") + + # throw error if not success + response.raise_for_status() + + # parse json + data = response.json() + + # return first commit id + return data[0]["sha"] + + raise ValueError(f"Unsupported: {url}") + +def main(): + # load options + config = configparser.ConfigParser() + config.read(".ci/options.ini") + + # default option value + config["revisions"] = {} + + # iterate over repositories + for key, value in config.items("repositories"): + config["revisions"][key] = fetch_latest_revision(value) + + # write config back + with open(".ci/options.ini", "w") as file: + config.write(file) + +if __name__ == "__main__": + main() diff --git a/.github/workflows/gh-build-iso-cuda-gui.yml b/.github/workflows/gh-build-iso-cuda-gui.yml new file mode 100644 index 0000000..716ad4d --- /dev/null +++ b/.github/workflows/gh-build-iso-cuda-gui.yml @@ -0,0 +1,63 @@ +name: Build ISO on hosted runner (CUDA, gui) + +on: + push: + branches: + - main + + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Cleanup + uses: rokibhasansagar/slimhub_actions@main + with: + retain: "docker_buildkit,docker_imgcache" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build image + uses: addnab/docker-run-action@v3 + with: + image: archlinux:latest + + options: --privileged --volume ${{ github.workspace }}:/workspace + run: | + # Exit on error + set -eu + + # Enter project directory + pushd /workspace + # Install dependencies + .ci/dependencies.sh + + # Patch mkarchiso + .ci/mkarchiso.sh + + + # Configure to use CUDA + .ci/configure.py cuda gui + popd + + # Build image + mkarchiso -v -m iso -w /_work -o /workspace/out /workspace + + - name: Create summary + run: | + # Exit on error + set -eu + + # Print checksums to summary + sha256sum out/* > "$GITHUB_STEP_SUMMARY" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: archiso-output + path: out/ \ No newline at end of file diff --git a/.github/workflows/gh-build-iso-cuda.yml b/.github/workflows/gh-build-iso-cuda.yml index ecc11b3..b0cb789 100644 --- a/.github/workflows/gh-build-iso-cuda.yml +++ b/.github/workflows/gh-build-iso-cuda.yml @@ -21,8 +21,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - with: - submodules: recursive - name: Build image uses: addnab/docker-run-action@v3 diff --git a/.github/workflows/sh-build-iso-rocm-gui.yml b/.github/workflows/sh-build-iso-rocm-gui.yml new file mode 100644 index 0000000..c75b726 --- /dev/null +++ b/.github/workflows/sh-build-iso-rocm-gui.yml @@ -0,0 +1,145 @@ +name: Build ISO on selfhosted runner (ROCm, gui) + +on: + push: + branches: + - main + +jobs: + vm-start: + if: always() + runs-on: ubuntu-latest + + steps: + - name: Save ssh configuration + run: | + # Create ssh directory + mkdir ~/.ssh + + + # Save ssh config + echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config + + # Save ssh known hosts + echo "${{ secrets.SSH_KNOWN_HOSTS }}" | base64 -d > ~/.ssh/known_hosts + + # Save ssh private key + echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key + + # Fix permissions for private key + chmod 600 /tmp/private.key + + - name: Wait 30 seconds + run: | + sleep 30 + + - name: Wait until Github Actions Runner VM is stopped + run: | + ok="0" + + while true; do + status=$(ssh runner "virsh domstate GithubActions") + + if [ "$status" = "shutoff" ]; then + if [ "$ok" = "1" ]; then + echo "VM is shut off. Exiting." + break + else + sleep $((10 + $RANDOM % 10)) + ok="1" + fi + else + ok="0" + fi + + echo "VM is not shut off yet. Waiting for 5 seconds..." + sleep 5 + done + + - name: Start Github Actions Runner VM + run: | + ssh runner "virsh start GithubActions" + + build: + needs: vm-start + runs-on: self-hosted + + steps: + - name: Remove previous artifacts + run: | + sudo rm -fr out/ + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build image + uses: addnab/docker-run-action@v3 + with: + image: archlinux:latest + + options: --privileged --rm --volume ${{ github.workspace }}:/workspace + run: | + # Exit on error + set -eu + + # Enter project directory + pushd /workspace + # Install dependencies + .ci/dependencies.sh + + # Patch mkarchiso + .ci/mkarchiso.sh + + + # Configure to use ROCm + .ci/configure.py rocm gui + popd + + # Build image + mkarchiso -v -m iso -w /_work -o /workspace/out /workspace + + - name: Create summary + run: | + # Exit on error + set -eu + + # Print checksums to summary + sha256sum out/* > "$GITHUB_STEP_SUMMARY" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: archiso-output + path: out/ + + vm-stop: + if: always() + needs: build + runs-on: ubuntu-latest + + steps: + - name: Save ssh configuration + run: | + # Create ssh directory + mkdir ~/.ssh + + + # Save ssh config + echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config + + # Save ssh known hosts + echo "${{ secrets.SSH_KNOWN_HOSTS }}" | base64 -d > ~/.ssh/known_hosts + + # Save ssh private key + echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key + + # Fix permissions for private key + chmod 600 /tmp/private.key + + - name: Stop Github Actions Runner VM + run: | + # Invoke regular shutdown + ssh runner "virsh shutdown GithubActions" + + # Wait 2 minutes + sleep 120 \ No newline at end of file diff --git a/.github/workflows/sh-build-iso-rocm.yml b/.github/workflows/sh-build-iso-rocm.yml index eca8382..438226a 100644 --- a/.github/workflows/sh-build-iso-rocm.yml +++ b/.github/workflows/sh-build-iso-rocm.yml @@ -5,10 +5,6 @@ on: branches: - main - pull_request: - branches: - - main - jobs: vm-start: if: always() @@ -33,6 +29,33 @@ jobs: # Fix permissions for private key chmod 600 /tmp/private.key + - name: Wait 0 seconds + run: | + sleep 0 + + - name: Wait until Github Actions Runner VM is stopped + run: | + ok="0" + + while true; do + status=$(ssh runner "virsh domstate GithubActions") + + if [ "$status" = "shutoff" ]; then + if [ "$ok" = "1" ]; then + echo "VM is shut off. Exiting." + break + else + sleep $((10 + $RANDOM % 10)) + ok="1" + fi + else + ok="0" + fi + + echo "VM is not shut off yet. Waiting for 5 seconds..." + sleep 5 + done + - name: Start Github Actions Runner VM run: | ssh runner "virsh start GithubActions" @@ -48,8 +71,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - with: - submodules: recursive - name: Build image uses: addnab/docker-run-action@v3 @@ -121,7 +142,4 @@ jobs: ssh runner "virsh shutdown GithubActions" # Wait 2 minutes - sleep 120 - - # Force terminate if needed - ssh runner "virsh destroy GithubActions" || true \ No newline at end of file + sleep 120 \ No newline at end of file diff --git a/.github/workflows/sh-do-nothing.yml b/.github/workflows/sh-do-nothing.yml new file mode 100644 index 0000000..a0cee07 --- /dev/null +++ b/.github/workflows/sh-do-nothing.yml @@ -0,0 +1,78 @@ +name: Do nothing on selfhosted runner + +on: + schedule: + - cron: "30 1 1,15 * *" + + workflow_dispatch: + +jobs: + vm-start: + runs-on: ubuntu-latest + + steps: + - name: Save ssh configuration + run: | + # Create ssh directory + mkdir ~/.ssh + + # Save ssh config + echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config + + # Save ssh known hosts + echo "${{ secrets.SSH_KNOWN_HOSTS }}" | base64 -d > ~/.ssh/known_hosts + + # Save ssh private key + echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key + + # Fix permissions for private key + chmod 600 /tmp/private.key + + - name: Start Github Actions Runner VM + run: | + status=$(ssh runner "virsh domstate GithubActions") + + if [ "$status" = "shutoff" ]; then + ssh runner "virsh start GithubActions" + echo "started=true" >> "$GITHUB_OUTPUT" + else + echo "started=false" >> "$GITHUB_OUTPUT" + fi + + do-nothing: + if: ${{ needs.vm-start.outputs.started == 'true' }} + needs: vm-start + runs-on: self-hosted + + steps: + - name: Do nothing + run: | + true + + vm-stop: + if: ${{ needs.vm-start.outputs.started == 'true' }} + needs: do-nothing + runs-on: ubuntu-latest + + steps: + - name: Save ssh configuration + run: | + # Create ssh directory + mkdir ~/.ssh + + # Save ssh config + echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config + + # Save ssh known hosts + echo "${{ secrets.SSH_KNOWN_HOSTS }}" | base64 -d > ~/.ssh/known_hosts + + # Save ssh private key + echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key + + # Fix permissions for private key + chmod 600 /tmp/private.key + + - name: Stop Github Actions Runner VM + run: | + # Invoke regular shutdown + ssh runner "virsh shutdown GithubActions" diff --git a/.gitignore b/.gitignore index bb5d104..a66378d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,14 @@ work/ airootfs/root/customize_airootfs.sh airootfs/root/customize_airootfs/scripts/1000-automatic.sh airootfs/root/customize_airootfs/scripts/1000-axolotl.sh -airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh +airootfs/root/customize_airootfs/scripts/1000-comfyui.sh airootfs/root/customize_airootfs/scripts/1000-koboldcpp.sh airootfs/root/customize_airootfs/scripts/1000-llamacpp.sh -airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh -airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh +airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras.sh +airootfs/root/customize_airootfs/scripts/1000-text-generation-webui.sh +airootfs/root/customize_airootfs/scripts/1000-vllm.sh +airootfs/root/customize_airootfs/scripts/9000-misc.sh airootfs/root/customize_airootfs/scripts/9999-cleanup.sh packages.x86_64 +pacman.conf profiledef.sh diff --git a/README.md b/README.md index 10d74f9..ad53691 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,17 @@ # ToriLinux - Linux LiveCD for offline AI training and inference +![vLLM on ToriLinux](docs/media/vllm-on-torilinux.png) + LiveCD distribution based on [ArchLinux](https://archlinux.org/) and currently includes the following projects preinstalled, along with their dependencies: * [automatic](https://github.com/vladmandic/automatic) * [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) * [ComfyUI](https://github.com/comfyanonymous/ComfyUI) * [koboldcpp](https://github.com/LostRuins/koboldcpp) * [llama.cpp](https://github.com/ggerganov/llama.cpp) +* [SillyTavern](https://github.com/SillyTavern/SillyTavern) * [SillyTavern-Extras](https://github.com/SillyTavern/SillyTavern-Extras) * [text-generation-webui](https://github.com/oobabooga/text-generation-webui) +* [vllm](https://github.com/vllm-project/vllm) If you would like to see another AI-related project included in ToriLinux, please open an [issue](https://github.com/sasha0552/ToriLinux/issues/new). @@ -31,5 +35,6 @@ Note that you need pre-downloaded models on a local hard drive or NFS server, or Note that following projects is not available on ROCm version: * [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) +* [vllm](https://github.com/vllm-project/vllm) The server for building the ROCm version is provided by [@Sepera-okeq](https://github.com/Sepera-okeq/). diff --git a/airootfs/etc/X11/xorg.conf b/airootfs/etc/X11/xorg.conf new file mode 100644 index 0000000..d90ce04 --- /dev/null +++ b/airootfs/etc/X11/xorg.conf @@ -0,0 +1,4 @@ +Section "Device" + Identifier "modesetting" + Driver "modesetting" +EndSection diff --git a/airootfs/etc/lsb-release b/airootfs/etc/lsb-release deleted file mode 100644 index d4e896e..0000000 --- a/airootfs/etc/lsb-release +++ /dev/null @@ -1,3 +0,0 @@ -DISTRIB_ID="Tori" -DISTRIB_RELEASE="rolling" -DISTRIB_DESCRIPTION="Tori Linux" diff --git a/airootfs/etc/nodm.conf b/airootfs/etc/nodm.conf new file mode 100644 index 0000000..373df36 --- /dev/null +++ b/airootfs/etc/nodm.conf @@ -0,0 +1,23 @@ +# nodm configuration file + +# Controls the user that is used to automatically log in +NODM_USER='tori' + +# Options to pass to the X server (for example: "vt7 -nolisten +# tcp") +NODM_X_OPTIONS='vt7 -nolisten tcp' + +# Minimum time (in seconds) that a session should last in order +# for nodm to decide that it has not quit too soon. If an X +# session will run for less than this time, nodm will wait an +# increasing bit of time before restarting it. +NODM_MIN_SESSION_TIME=60 + +# X session command (default: /etc/X11/Xsession). It is run using +# the shell, so it can be any shell command. +NODM_XSESSION='/home/tori/.xinitrc' + +# Timeout (in seconds) to wait for X to be ready to accept +# connections. If X is not ready before this timeout, it is killed +# and restarted. +NODM_X_TIMEOUT=20 diff --git a/airootfs/etc/systemd/journald.conf b/airootfs/etc/systemd/journald.conf deleted file mode 100644 index 74beb58..0000000 --- a/airootfs/etc/systemd/journald.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Journal] -Storage=none diff --git a/airootfs/etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service b/airootfs/etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service deleted file mode 120000 index 28c0a35..0000000 --- a/airootfs/etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service +++ /dev/null @@ -1 +0,0 @@ -/usr/lib/systemd/system/nvidia-persistenced.service \ No newline at end of file diff --git a/airootfs/home/tori/.config/openbox/autostart b/airootfs/home/tori/.config/openbox/autostart new file mode 100755 index 0000000..248edff --- /dev/null +++ b/airootfs/home/tori/.config/openbox/autostart @@ -0,0 +1,2 @@ +feh --bg-scale ~/.local/share/tori/wallpapers/tori-aod.jpg & +tint2 & diff --git a/airootfs/home/tori/.config/openbox/menu.xml b/airootfs/home/tori/.config/openbox/menu.xml new file mode 100644 index 0000000..c7b5740 --- /dev/null +++ b/airootfs/home/tori/.config/openbox/menu.xml @@ -0,0 +1,82 @@ + + + + + + + kmahjongg + + + + + + kmines + + + + + + kpat + + + + + + ksudoku + + + + + + + + firefox + + + + + + + + konsole + + + + + + + + konsole -e btop + + + + + + konsole -e htop + + + + + + konsole -e nvtop + + + + + + + + + + + + + + + + + + + + + diff --git a/airootfs/root/customize_airootfs/patches/0000-automatic-drop-pstate-in-idle.patch b/airootfs/home/tori/.local/share/tori/patches/0000-automatic-drop-pstate-in-idle.patch similarity index 81% rename from airootfs/root/customize_airootfs/patches/0000-automatic-drop-pstate-in-idle.patch rename to airootfs/home/tori/.local/share/tori/patches/0000-automatic-drop-pstate-in-idle.patch index 8769a51..daf7201 100644 --- a/airootfs/root/customize_airootfs/patches/0000-automatic-drop-pstate-in-idle.patch +++ b/airootfs/home/tori/.local/share/tori/patches/0000-automatic-drop-pstate-in-idle.patch @@ -8,15 +8,7 @@ debug_install = installer.log.debug if os.environ.get('SD_INSTALL_DEBUG', None) is not None else lambda *args, **kwargs: None -@@ -169,6 +170,7 @@ def start_server(immediate=True, server=None): - uvicorn = None - if args.test: - installer.log.info("Test only") -+ exit(0) - server.wants_restart = False - else: - if args.api_only: -@@ -260,4 +262,5 @@ def main(): +@@ -260,4 +261,5 @@ def main(): if __name__ == "__main__": diff --git a/airootfs/root/customize_airootfs/patches/0000-koboldcpp-drop-pstate-in-idle.patch b/airootfs/home/tori/.local/share/tori/patches/0000-koboldcpp-drop-pstate-in-idle.patch similarity index 88% rename from airootfs/root/customize_airootfs/patches/0000-koboldcpp-drop-pstate-in-idle.patch rename to airootfs/home/tori/.local/share/tori/patches/0000-koboldcpp-drop-pstate-in-idle.patch index 8e316cd..be7c0ae 100644 --- a/airootfs/root/customize_airootfs/patches/0000-koboldcpp-drop-pstate-in-idle.patch +++ b/airootfs/home/tori/.local/share/tori/patches/0000-koboldcpp-drop-pstate-in-idle.patch @@ -8,7 +8,7 @@ sampler_order_max = 7 stop_token_max = 16 -@@ -397,6 +398,7 @@ def load_model(model_filename): +@@ -399,6 +400,7 @@ def load_model(model_filename): return ret def generate(prompt, memory="", images=[], max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}): @@ -16,7 +16,7 @@ global maxctx, args, currentusergenkey, totalgens, pendingabortkey inputs = generation_inputs() inputs.prompt = prompt.encode("UTF-8") -@@ -489,6 +491,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 +@@ -491,6 +493,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 if pendingabortkey!="" and pendingabortkey==genkey: print(f"\nDeferred Abort for GenKey: {pendingabortkey}") pendingabortkey = "" @@ -24,7 +24,7 @@ return "" else: ret = handle.generate(inputs) -@@ -500,6 +503,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 +@@ -502,6 +505,7 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512 sindex = outstr.find(trim_str) if sindex != -1 and trim_str!="": outstr = outstr[:sindex] @@ -32,7 +32,7 @@ return outstr -@@ -2679,6 +2683,7 @@ def sanitize_string(input_string): +@@ -2758,6 +2762,7 @@ def sanitize_string(input_string): return sanitized_string def main(launch_args,start_server=True): diff --git a/airootfs/home/tori/.local/share/tori/patches/0000-llamacpp-server-drop-pstate-in-idle.patch b/airootfs/home/tori/.local/share/tori/patches/0000-llamacpp-server-drop-pstate-in-idle.patch new file mode 100644 index 0000000..a9e4940 --- /dev/null +++ b/airootfs/home/tori/.local/share/tori/patches/0000-llamacpp-server-drop-pstate-in-idle.patch @@ -0,0 +1,26 @@ +--- a/examples/server/server.cpp ++++ b/examples/server/server.cpp +@@ -1039,6 +1039,7 @@ struct server_context { + {"id_task", slot.id_task}, + }); + ++ system("nvidia-pstate -s -ps 16"); + return true; + } + +@@ -1788,6 +1789,7 @@ struct server_context { + kv_cache_clear(); + } + ++ system("nvidia-pstate -s -ps 8"); + return; + } + } +@@ -2895,6 +2897,7 @@ inline void signal_handler(int signal) { + } + + int main(int argc, char ** argv) { ++ system("nvidia-pstate -s -ps 8"); + #if SERVER_VERBOSE != 1 + log_disable(); + #endif diff --git a/airootfs/home/tori/.local/share/tori/patches/0100-vllm-enable-other-archs.patch b/airootfs/home/tori/.local/share/tori/patches/0100-vllm-enable-other-archs.patch new file mode 100644 index 0000000..486f379 --- /dev/null +++ b/airootfs/home/tori/.local/share/tori/patches/0100-vllm-enable-other-archs.patch @@ -0,0 +1,11 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -16,7 +16,7 @@ + set(PYTHON_SUPPORTED_VERSIONS "3.8" "3.9" "3.10" "3.11") + + # Supported NVIDIA architectures. +-set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0") ++set(CUDA_SUPPORTED_ARCHS "6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0") + + # Supported AMD GPU architectures. + set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100") diff --git a/airootfs/home/tori/.local/share/tori/wallpapers/tori-aod.jpg b/airootfs/home/tori/.local/share/tori/wallpapers/tori-aod.jpg new file mode 100644 index 0000000..9284c69 Binary files /dev/null and b/airootfs/home/tori/.local/share/tori/wallpapers/tori-aod.jpg differ diff --git a/airootfs/home/tori/.xinitrc b/airootfs/home/tori/.xinitrc new file mode 100644 index 0000000..2c21291 --- /dev/null +++ b/airootfs/home/tori/.xinitrc @@ -0,0 +1 @@ +exec openbox-session diff --git a/airootfs/root/customize_airootfs.sh.jinja2 b/airootfs/root/customize_airootfs.sh.jinja2 index f671311..b05f0f9 100644 --- a/airootfs/root/customize_airootfs.sh.jinja2 +++ b/airootfs/root/customize_airootfs.sh.jinja2 @@ -1,8 +1,33 @@ #!/bin/bash set -eu -# move new os-release to its place -mv /usr/lib/os-release.new /usr/lib/os-release +##### ENVIRONMENT VARIABLES ##### +# customize_airootfs temporary directory +export CUSTOMIZE_AIROOTFS=/root/customize_airootfs + +# directory with patches +export TORI_PATCHES=/home/tori/.local/share/tori/patches + +# disable package caching +export PIP_NO_CACHE_DIR=0 + +# limit the number of parallel jobs to avoid OOM +export MAX_JOBS=2 + +# define supported architectures +export TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0" +export CMAKE_CUDA_ARCHITECTURES=$TORCH_CUDA_ARCH_LIST +export CUDA_ARCHITECTURES=$TORCH_CUDA_ARCH_LIST + +# cuda home directory +export CUDA_HOME=/opt/cuda + +# use gcc 12 +export CC=gcc-12 +export CXX=g++-12 +export CUDACXX=$CUDA_HOME/bin/nvcc +export CUDAHOSTCXX=g++-12 +##### ENVIRONMENT VARIABLES ##### # set user password echo "tori:tori" | chpasswd @@ -11,17 +36,17 @@ echo "tori:tori" | chpasswd find /root -type f -name "*.jinja2" -print -delete {% if platform == "cuda" %} -# install nvidia-pstate if cuda +# install nvidia-pstate pip3 install --break-system-packages nvidia-pstate -{% endif %} -{% if platform == "rocm" %} -# remove nvidia-persistenced if rocm -rm -f /etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service +# enable nvidia-persistenced service +systemctl enable nvidia-persistenced {% endif %} -# customize_airootfs temporary directory -export CUSTOMIZE_AIROOTFS=/root/customize_airootfs +{% if type == "gui" %} +# enable nodm +systemctl enable nodm +{% endif %} {% if type != "empty" %} # enter home directory diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-automatic.sh.jinja2 index 815fa44..3d4fd55 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-automatic.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-automatic.sh.jinja2 @@ -2,55 +2,35 @@ set -eu # clone repository -git clone "https://github.com/vladmandic/automatic.git" +git clone "{{ repositories.automatic }}" # automatic patches pushd "automatic" # use specific revision - git checkout e783b098fd0ad28acabb29844ab0af0978684c04 + git checkout "{{ revisions.automatic }}" # clone submodules git submodule update --init --recursive - # remove git dependency - sed -i '/installer.check_modified_files()/d' launch.py - sed -i '/installer.install_submodules()/d' launch.py - sed -i '/installer.update_wiki()/d' launch.py - sed -i 's/installer.check_timestamp()/False/g' launch.py - - # remove internet dependency - sed -i '/installer.check_version()/d' launch.py - sed -i 's/lambda: {"choices": theme.list_themes()}, refresh=theme.refresh_themes/{"choices": ["black-teal"]}/g' modules/shared.py - sed -i 's/shared.opts.motd/False/g' modules/api/api.py - -{% if platform == "cuda" %} + {% if platform == "cuda" %} # drop pstate in idle - patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0000-automatic-drop-pstate-in-idle.patch" -{% endif %} + patch -p1 < "$TORI_PATCHES/0000-automatic-drop-pstate-in-idle.patch" + {% endif %} popd # automatic dependencies pushd "automatic" - # disable package caching - export PIP_NO_CACHE_DIR=0 - # create venv python3 -m venv venv # activate venv source venv/bin/activate {% if platform == "cuda" %} - # install nvidia-pstate if cuda + # install nvidia-pstate pip3 install nvidia-pstate {% endif %} # install dependencies python3 launch.py --test deactivate - - # remove installation config - rm config.json - - # remove installation log - rm sdnext.log popd diff --git a/airootfs/root/customize_airootfs/scripts/1000-axolotl.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-axolotl.sh.jinja2 index e8e1188..1acb8f1 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-axolotl.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-axolotl.sh.jinja2 @@ -3,37 +3,26 @@ set -eu {% if platform == "cuda" %} # clone repository -git clone "https://github.com/OpenAccess-AI-Collective/axolotl.git" +git clone "{{ repositories.axolotl }}" # axolotl patches pushd "axolotl" # use specific revision - git checkout 2fa65b95997fa0ae68ae2b66933521ff8dfa3980 + git checkout "{{ revisions.axolotl }}" popd # axolotl dependencies pushd "axolotl" - # disable package caching - export PIP_NO_CACHE_DIR=0 - - # limit the number of parallel jobs to avoid OOM - export MAX_JOBS=1 - - # define supported architectures - export TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0" - - # cuda home directory - export CUDA_HOME=/opt/cuda - - # use gcc 12 - export CC=gcc-12 - export CXX=g++-12 - # create venv python3 -m venv venv # activate venv source venv/bin/activate + {% if platform == "cuda" %} + # install nvidia-pstate + pip3 install nvidia-pstate + {% endif %} + # install dependencies # TODO: install specific torch version pip3 install packaging torch==2.1.1 wheel diff --git a/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-comfyui.sh.jinja2 similarity index 68% rename from airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2 rename to airootfs/root/customize_airootfs/scripts/1000-comfyui.sh.jinja2 index 255d656..baf49f0 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-comfyui.sh.jinja2 @@ -2,31 +2,33 @@ set -eu # clone repository -git clone "https://github.com/comfyanonymous/ComfyUI.git" +git clone "{{ repositories.comfyui }}" # ComfyUI patches pushd "ComfyUI" # use specific revision - git checkout 30abc324c2f73e6b648093ccd4741dece20be1e5 + git checkout "{{ revisions.comfyui }}" popd # ComfyUI dependencies pushd "ComfyUI" - # disable package caching - export PIP_NO_CACHE_DIR=0 - # create venv python3 -m venv venv # activate venv source venv/bin/activate -{% if platform == "rocm" %} + {% if platform == "cuda" %} + # install nvidia-pstate + pip3 install nvidia-pstate + {% endif %} + + {% if platform == "rocm" %} # extract pytorch version index_url=$(grep -o 'https://download.pytorch.org/whl/rocm[0-9.]*' README.md) # install pytorch pip3 install torch torchvision torchaudio --index-url "$index_url" -{% endif %} + {% endif %} # install dependencies pip3 install -r requirements.txt diff --git a/airootfs/root/customize_airootfs/scripts/1000-koboldcpp.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-koboldcpp.sh.jinja2 index 59d5a81..78c8289 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-koboldcpp.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-koboldcpp.sh.jinja2 @@ -2,15 +2,15 @@ set -eu # clone repository -git clone "https://github.com/LostRuins/koboldcpp.git" +git clone "{{ repositories.koboldcpp }}" # koboldcpp patches pushd "koboldcpp" # use specific revision - git checkout f3b7651102c3ce3e4f331b93137dc32d752eada0 + git checkout "{{ revisions.koboldcpp }}" -{% if platform == "cuda" %} + {% if platform == "cuda" %} # drop pstate in idle - patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0000-koboldcpp-drop-pstate-in-idle.patch" -{% endif %} + patch -p1 < "$TORI_PATCHES/0000-koboldcpp-drop-pstate-in-idle.patch" + {% endif %} popd diff --git a/airootfs/root/customize_airootfs/scripts/1000-llamacpp.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-llamacpp.sh.jinja2 index a5da2cd..0a7aecd 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-llamacpp.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-llamacpp.sh.jinja2 @@ -2,10 +2,15 @@ set -eu # clone repository -git clone "https://github.com/ggerganov/llama.cpp.git" +git clone "{{ repositories.llamacpp }}" # llama.cpp patches pushd "llama.cpp" # use specific revision - git checkout cc4a95426d17417d3c83f12bdb514fbe8abe2a88 + git checkout "{{ revisions.llamacpp }}" + + {% if platform == "cuda" %} + # drop pstate in idle + patch -p1 < "$TORI_PATCHES/0000-llamacpp-server-drop-pstate-in-idle.patch" + {% endif %} popd diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras.sh.jinja2 similarity index 64% rename from airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 rename to airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras.sh.jinja2 index f017703..8111011 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras.sh.jinja2 @@ -2,33 +2,35 @@ set -eu # clone repository -git clone "https://github.com/SillyTavern/SillyTavern-Extras.git" +git clone "{{ repositories.sillytavern_extras }}" # SillyTavern-Extras patches pushd "SillyTavern-Extras" # use specific revision - git checkout 86793c6e104ce7e69da919eecea7f4d87ed61727 + git checkout "{{ revisions.sillytavern_extras }}" popd # SillyTavern-Extras dependencies pushd "SillyTavern-Extras" - # disable package caching - export PIP_NO_CACHE_DIR=0 - # create venv python3 -m venv venv # activate venv source venv/bin/activate -{% if platform == "cuda" %} + {% if platform == "cuda" %} + # install nvidia-pstate + pip3 install nvidia-pstate + {% endif %} + + {% if platform == "cuda" %} # install dependencies (cuda) pip3 install -r requirements.txt -{% endif %} + {% endif %} -{% if platform == "rocm" %} + {% if platform == "rocm" %} # install dependencies (rocm) pip3 install -r requirements-rocm.txt -{% endif %} + {% endif %} # install remaining dependencies pip3 install -r requirements-coqui.txt diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern.sh.jinja2 new file mode 100644 index 0000000..8440036 --- /dev/null +++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern.sh.jinja2 @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +# clone repository +git clone "{{ repositories.sillytavern }}" + +# SillyTavern patches +pushd "SillyTavern" + # use specific revision + git checkout "{{ revisions.sillytavern }}" +popd + +# SillyTavern dependencies +pushd "SillyTavern" + # install dependencies + npm install +popd diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui.sh.jinja2 similarity index 68% rename from airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 rename to airootfs/root/customize_airootfs/scripts/1000-text-generation-webui.sh.jinja2 index 05e45bc..ed1a985 100644 --- a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui.sh.jinja2 @@ -2,30 +2,32 @@ set -eu # clone repository -git clone "https://github.com/oobabooga/text-generation-webui.git" +git clone "{{ repositories.text_generation_webui }}" # text-generation-webui patches pushd "text-generation-webui" # use specific revision - git checkout 91a7370a655881c55274284509a546ffd644dc16 + git checkout "{{ revisions.text_generation_webui }}" popd # text-generation-webui dependencies pushd "text-generation-webui" - # disable package caching - export PIP_NO_CACHE_DIR=0 - # create venv python3 -m venv venv # activate venv source venv/bin/activate -{% if platform == "cuda" %} + {% if platform == "cuda" %} + # install nvidia-pstate + pip3 install nvidia-pstate + {% endif %} + + {% if platform == "cuda" %} # install dependencies (cuda) pip3 install -r requirements.txt -{% endif %} + {% endif %} -{% if platform == "rocm" %} + {% if platform == "rocm" %} # extract pytorch version index_url=$(grep -m1 -o 'https://download.pytorch.org/whl/rocm[0-9.]*' one_click.py) @@ -34,6 +36,6 @@ pushd "text-generation-webui" # install dependencies (rocm) pip3 install -r requirements_amd.txt -{% endif %} + {% endif %} deactivate popd diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm.sh.jinja2 new file mode 100644 index 0000000..143fefc --- /dev/null +++ b/airootfs/root/customize_airootfs/scripts/1000-vllm.sh.jinja2 @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +{% if platform == "cuda" %} +# clone repository +git clone "{{ repositories.vllm }}" + +# vllm patches +pushd "vllm" + # use specific revision + git checkout "{{ revisions.vllm }}" + + # enable other archs + patch -p1 < "$TORI_PATCHES/0100-vllm-enable-other-archs.patch" +popd + +# vllm dependencies +pushd "vllm" + # create venv + python3 -m venv venv + + # activate venv + source venv/bin/activate + # install dependencies + pip3 install -r requirements-build.txt + + # build native extension + python3 setup.py build_ext --inplace + deactivate + + # remove venv + rm -fr venv + + # create venv + python3 -m venv venv + + # activate venv + source venv/bin/activate + {% if platform == "cuda" %} + # install nvidia-pstate + pip3 install nvidia-pstate + {% endif %} + + # install dependencies + pip3 install -r requirements-cuda.txt + deactivate +popd +{% endif %} diff --git a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 index 4b6f6d5..c1ef39f 100644 --- a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 +++ b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 @@ -9,3 +9,9 @@ rm -fr /home/tori/.config/matplotlib # keras rm -fr /home/tori/.keras + +# automatic installation config +rm -f /home/tori/automatic/config.json + +# automatic installation log +rm -f /home/tori/automatic/sdnext.log diff --git a/airootfs/usr/lib/os-release.new b/airootfs/usr/lib/os-release.new deleted file mode 100644 index 8442993..0000000 --- a/airootfs/usr/lib/os-release.new +++ /dev/null @@ -1,11 +0,0 @@ -NAME="Tori Linux" -PRETTY_NAME="Tori Linux" -ID=tori -BUILD_ID=rolling -ANSI_COLOR="38;2;23;147;209" -HOME_URL="https://github.com/sasha0552/ToriLinux" -DOCUMENTATION_URL="https://github.com/sasha0552/ToriLinux" -SUPPORT_URL="https://github.com/sasha0552/ToriLinux/issues" -BUG_REPORT_URL="https://github.com/sasha0552/ToriLinux/issues" -PRIVACY_POLICY_URL="https://github.com/sasha0552/ToriLinux" -LOGO=torilinux-logo diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..05a4f59 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# ToriLinux Documentation + +TODO diff --git a/docs/media/vllm-on-torilinux.png b/docs/media/vllm-on-torilinux.png new file mode 100644 index 0000000..187e30b Binary files /dev/null and b/docs/media/vllm-on-torilinux.png differ diff --git a/packages.x86_64.jinja2 b/packages.x86_64.jinja2 index 7c01377..f02e20e 100644 --- a/packages.x86_64.jinja2 +++ b/packages.x86_64.jinja2 @@ -22,6 +22,8 @@ ncdu neofetch net-tools nfs-utils +nodejs +npm ntfs-3g nvtop openssh @@ -50,3 +52,24 @@ rocm-hip-sdk # Required for SillyTavern-Extras gtk3 {% endif %} + +{% if type == "gui" %} +# GUI +feh +nodm +obconf +openbox +tint2 +ttf-dejavu +ttf-liberation +xorg +xorg-xinit + +# GUI applications +firefox +kmahjongg +kmines +konsole +kpat +ksudoku +{% endif %} diff --git a/pacman.conf b/pacman.conf.jinja2 similarity index 89% rename from pacman.conf rename to pacman.conf.jinja2 index f382fab..bfae9c6 100644 --- a/pacman.conf +++ b/pacman.conf.jinja2 @@ -74,13 +74,23 @@ LocalFileSigLevel = Optional #Include = /etc/pacman.d/mirrorlist [core] +{% if platform == "cuda" %} +# TODO: https://github.com/pytorch/pytorch/issues/122169 +Server = https://archive.archlinux.org/repos/2024/03/04/$repo/os/$arch +{% else %} Include = /etc/pacman.d/mirrorlist +{% endif %} #[extra-testing] #Include = /etc/pacman.d/mirrorlist [extra] +{% if platform == "cuda" %} +# TODO: https://github.com/pytorch/pytorch/issues/122169 +Server = https://archive.archlinux.org/repos/2024/03/04/$repo/os/$arch +{% else %} Include = /etc/pacman.d/mirrorlist +{% endif %} # If you want to run 32 bit applications on your x86_64 system, # enable the multilib repositories as required here.