infra: update aggregation mode deployment

.github/workflows/aggregation_mode.yml

@@ -0,0 +1,46 @@
+name: "Start Aggregation Mode Server"
+
+# Starts the Paperspace GPU server that runs the aggregation mode.
+#
+# The server is kept powered off to avoid 24/7 billing. This workflow boots it
+# once a day; on boot the machine runs `aggregation_mode.service`, which executes
+# the SP1 aggregations and then powers the machine off again
+# (see infra/aggregation_mode/run.sh).
+on:
+ schedule:
+ # 15:00 UTC == 12:00 GMT-3, every day. GitHub Actions cron is always in UTC.
+ - cron: "0 15 * * *"
+ workflow_dispatch:
+
+jobs:
+ start-server:
+ name: Start Paperspace aggregation server
+ runs-on: ubuntu-latest
+ steps:
+ - name: Start Paperspace machine
+ env:
+ PAPERSPACE_API_KEY: ${{ secrets.PAPERSPACE_API_KEY }}
+ MACHINE_ID: ${{ secrets.PAPERSPACE_MACHINE_ID }}
+ run: |
+ set -euo pipefail
+
+ if [ -z "${PAPERSPACE_API_KEY}" ] || [ -z "${MACHINE_ID}" ]; then
+ echo "::error::PAPERSPACE_API_KEY and PAPERSPACE_MACHINE_ID secrets must be set."
+ exit 1
+ fi
+
+ echo "Starting Paperspace machine ${MACHINE_ID}..."
+ http_code=$(curl -sS -o response.json -w "%{http_code}" \
+ -X PATCH "https://api.paperspace.com/v1/machines/${MACHINE_ID}/start" \
+ -H "Authorization: Bearer ${PAPERSPACE_API_KEY}")
+
+ echo "Paperspace API responded with HTTP ${http_code}"
+ cat response.json || true
+
+ # 2xx means the start request was accepted.
+ if [ "${http_code}" -lt 200 ] || [ "${http_code}" -ge 300 ]; then
+ echo "::error::Failed to start Paperspace machine (HTTP ${http_code})."
+ exit 1
+ fi
+
+ echo "Start request accepted. The machine will run the aggregation on boot and shut itself down afterwards."

infra/aggregation_mode/README.md

@@ -1,19 +1,33 @@
 # Aggregation Mode Setup
 
+## Overview
+
+The aggregation mode runs on a Paperspace GPU server. To avoid paying for the GPU
+24/7, the server is kept powered off and is only started once a day:
+
+1. The [`Start Aggregation Mode Server`](../../.github/workflows/aggregation_mode.yml)
+ GitHub Actions workflow runs daily at 15:00 UTC (12:00 GMT-3) and starts the
+ Paperspace machine via the Paperspace API.
+2. On boot, `aggregation_mode.service` runs automatically and executes
+ [`run.sh`](run.sh), which runs the SP1 and Risc0 aggregations.
+3. When the aggregations finish, `run.sh` powers the machine off again
+ (`sudo shutdown -h now`), so Paperspace stops billing it.
+
+The workflow needs:
+
+- `PAPERSPACE_API_KEY` repository **secret** — a Paperspace API key.
+- `PAPERSPACE_MACHINE_ID` repository **secret** — the id of the GPU machine.
+
 ## Setup on Server with GPU
 
 To setup the server with GPU, follow the steps in [aggregation_mode.sh](aggregation_mode.sh).
 
-After running all the steps, `aggregation_mode.timer` will execute every 24hs the `aggregation_mode.service`
+After running all the steps, `aggregation_mode.service` is enabled to run on every
+boot. There is no timer anymore — the daily schedule is driven by the GitHub Actions
+workflow described above.
 
 ## Check Service Status
 
-To check the status of the timer, run:
-
-```bash
-systemctl status aggregation_mode.timer --user
-```
-
 To check the status of the service, run:
 
 ```bash
@@ -22,7 +36,7 @@ systemctl status aggregation_mode.service --user
 
 ## Start Service manually
 
-If you need to start the service manually, without waiting for the timer, run:
+If you need to start the service manually (the machine is already on), run:
 
 ```bash
 systemctl start aggregation_mode.service --user
@@ -33,7 +47,7 @@ systemctl start aggregation_mode.service --user
 To check the logs of the service, run:
 
 ```bash
-journalctl -xfeu aggregation_mode.service --user
+journalctl -xfeu aggregation_mode.service --user
 ```
 
 Note: You can add `-n <n_of_lines>` to limit the number of lines to show.

infra/aggregation_mode/aggregation_mode.service

@@ -9,4 +9,4 @@ ExecStartPre=sleep 60
 ExecStart=/home/user/run.sh
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target

infra/aggregation_mode/aggregation_mode.sh

@@ -24,7 +24,8 @@ done
 sudo loginctl enable-linger user
 
 # Install other dependencies
-sudo apt install -y gcc pkg-config libssl-dev build-essential apt-transport-https ca-certificates curl software-properties-common nvtop
+sudo apt install -y gcc pkg-config libssl-dev build-essential apt-transport-https ca-certificates curl software-properties-common nvtop clang
+sudo apt-get install -y protobuf-compiler
 
 # Install docker
 sudo apt-get update
@@ -52,14 +53,24 @@ sudo tailscale up --ssh --advertise-tags=tag:server && sudo tailscale set --auto
 # Install CUDA
 sudo add-apt-repository ppa:graphics-drivers/ppa
 sudo apt update
-sudo apt install nvidia-driver-570
+sudo apt install -y nvidia-driver-580
 
-# If see errors
-sudo apt-mark unhold cuda-drivers cuda-toolkit-12-6 nvidia-dkms-565-server nvidia-fabricmanager-565 nvidia-headless-565-server nvidia-utils-565-server
-sudo apt update
-sudo apt install nvidia-driver-570
 sudo apt autoremove
-sudo apt autoclean
+wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+sudo dpkg -i cuda-keyring_1.1-1_all.deb
+sudo apt update
+sudo apt install -y cuda-toolkit-12-9
+## Add to ~/.bashrc
+export PATH=/usr/local/cuda-12.9/bin${PATH:+:${PATH}}
+export LD_LIBRARY_PATH=/usr/local/cuda-12.9/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+
+# If see errors
+#sudo apt-mark unhold cuda-drivers cuda-toolkit-12-6 nvidia-dkms-565-server nvidia-fabricmanager-565 nvidia-headless-565-server nvidia-utils-565-server
+#sudo apt update
+#sudo apt install nvidia-driver-570
+#sudo apt autoremove
+#sudo apt autoclean
+
 sudo reboot
 nvidia-smi # To check if the driver is installed correctly
 
@@ -108,26 +119,33 @@ read -p "Enter a block number for SP1 (last_aggregated_block): " num && echo "{\
 ./infra/aggregation_mode/config_file.sh ./infra/aggregation_mode/config-proof-aggregator-risc0.template.yaml $HOME/config/config-proof-aggregator-risc0.yaml
 read -p "Enter a block number for Risc0 (last_aggregated_block): " num && echo "{\"last_aggregated_block\":$num}" > $HOME/config/proof-aggregator-risc0.last_aggregated_block.json
 
+mkdir -p $HOME/repos/aggregation_mode/mainnet
+cd $HOME/repos/aggregation_mode/mainnet
+git clone https://github.com/yetanotherco/aligned_layer.git
+cd aligned_layer # Subsequent relative paths (cargo --path, ./infra/...) resolve from the repo root
 # Build the proof_aggregator
-make proof_aggregator_install
+#make proof_aggregator_install
+cargo install --path aggregation_mode/proof_aggregator --features prove,gpu --bin proof_aggregator_gpu --locked
+cargo install --path aggregation_mode/proof_aggregator --features prove --bin proof_aggregator_cpu --locked # This builds risc0 with CPU but sp1 is runnable with GPU
 
 # Copy run script
 cp ./infra/aggregation_mode/run.sh $HOME/run.sh
 chmod 744 $HOME/run.sh
 
 # Setup systemd service
+# The service is enabled to run on every boot (no timer). The machine is started
+# once a day by the GitHub Actions workflow (.github/workflows/aggregation_mode.yml),
+# runs the aggregation on startup and then shuts itself down (see run.sh).
 cp ./infra/aggregation_mode/aggregation_mode.service $HOME/.config/systemd/user/aggregation_mode.service
-cp ./infra/aggregation_mode/aggregation_mode.timer $HOME/.config/systemd/user/aggregation_mode.timer
 
-#sudo systemctl enable aggregation_mode.service
-systemctl --user enable aggregation_mode.timer
-systemctl --user start aggregation_mode.timer
+systemctl --user daemon-reload
+systemctl --user enable aggregation_mode.service
 
 # Run the proof_aggregator manually if you want
 systemctl --user start aggregation_mode.service
 
-# Check timer status
-systemctl --user status aggregation_mode.timer
+# Check service status
+systemctl --user status aggregation_mode.service
 
 # Check logs
 journalctl -xfeu aggregation_mode.service --user -n10

infra/aggregation_mode/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SLEEP=5
+SLEEP=60
 
 echo "Starting Aggregation Mode in $SLEEP seconds..."
 sleep $SLEEP
@@ -13,3 +13,11 @@ echo "SP1 Aggregation Mode finished"
 echo "Starting Risc0 Aggregation Mode..."
 AGGREGATOR=risc0 /home/user/.cargo/bin/proof_aggregator_gpu /home/user/config/config-proof-aggregator-risc0.yaml
 echo "Risc0 Aggregation Mode finished"
+
+# Aggregation finished: power off the machine so Paperspace stops billing it.
+# The GitHub Actions workflow (.github/workflows/aggregation_mode.yml) starts the
+# machine again on the next scheduled run, and aggregation_mode.service runs this
+# script automatically on boot.
+echo "Aggregation finished, shutting down the machine in 1 minute..."
+sleep $SLEEP
+sudo shutdown -h now