From 9805c3774d876467c8b9b33fb078d93215883990 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Sun, 9 Mar 2025 10:57:38 +0530 Subject: [PATCH 01/18] flwr basic example --- flops/certs.yml | 98 +++++++++++++ flops/compose.yml | 185 +++++++++++++++++++++++++ flops/flower/application/__init__.py | 1 + flops/flower/application/client_app.py | 55 ++++++++ flops/flower/application/server_app.py | 31 +++++ flops/flower/application/task.py | 112 +++++++++++++++ flops/flower/pyproject.toml | 40 ++++++ 7 files changed, 522 insertions(+) create mode 100644 flops/certs.yml create mode 100644 flops/compose.yml create mode 100644 flops/flower/application/__init__.py create mode 100644 flops/flower/application/client_app.py create mode 100644 flops/flower/application/server_app.py create mode 100644 flops/flower/application/task.py create mode 100644 flops/flower/pyproject.toml diff --git a/flops/certs.yml b/flops/certs.yml new file mode 100644 index 00000000..863c702f --- /dev/null +++ b/flops/certs.yml @@ -0,0 +1,98 @@ +services: + gen-certs: + build: + context: . + pull: true + dockerfile_inline: | + FROM ubuntu:latest + + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + openssl + + WORKDIR /app/script + + ARG SUPERLINK_IP=127.0.0.1 + + COPY <<-EOF superlink-certificate.conf + [req] + default_bits = 4096 + prompt = no + default_md = sha256 + req_extensions = req_ext + distinguished_name = dn + + [dn] + C = US + O = Flower + CN = localhost + + [req_ext] + subjectAltName = @alt_names + + [alt_names] + DNS.0 = superlink + IP.1 = ::1 + IP.2 = $${SUPERLINK_IP} + EOF + + COPY --chmod=744 <<-'EOF' generate.sh + #!/bin/bash + # This script will generate all certificates if ca.crt does not exist + + set -e + cd "$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ + + CA_PASSWORD=notsafe + + # Generate directories if not exists + + generate () { + mkdir -p "$$1" + + if [ -f ""$$1"/ca.crt" ]; then + echo "Skipping certificate generation as they already exist." + return 0 + fi + + # Generate the root certificate authority key and certificate based on key + openssl genrsa -out "$$1"/ca.key 4096 + openssl req \ + -new \ + -x509 \ + -key "$$1"/ca.key \ + -sha256 \ + -subj "/C=DE/ST=HH/O=CA, Inc." \ + -days 365 -out "$$1"/ca.crt + + # Generate a new private key for the server + openssl genrsa -out "$$1"/server.key 4096 + + # Create a signing CSR + openssl req \ + -new \ + -key "$$1"/server.key \ + -out "$$1"/server.csr \ + -config ./script/"$$2" + + # Generate a certificate for the server + openssl x509 \ + -req \ + -in "$$1"/server.csr \ + -CA "$$1"/ca.crt \ + -CAkey "$$1"/ca.key \ + -CAcreateserial \ + -out "$$1"/server.pem \ + -days 365 \ + -sha256 \ + -extfile ./script/"$$2" \ + -extensions req_ext + } + generate superlink-certificates superlink-certificate.conf + EOF + + WORKDIR /app + + ENTRYPOINT ["./script/generate.sh"] + volumes: + - ./superlink-certificates/:/app/superlink-certificates/:rw diff --git a/flops/compose.yml b/flops/compose.yml new file mode 100644 index 00000000..9a19788e --- /dev/null +++ b/flops/compose.yml @@ -0,0 +1,185 @@ +services: + # create a SuperLink service + superlink: + image: flwr/superlink:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --isolation + - process + ports: + - 9093:9093 + + # create a ServerApp service + serverapp: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/serverapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-serverapp"] + command: + - --insecure + - --serverappio-api-address + - superlink:9091 + restart: on-failure:3 + depends_on: + - superlink + + # create two SuperNode services with different node configs + supernode-1: + image: flwr/supernode:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9094 + - --isolation + - process + - --node-config + - "partition-id=0 num-partitions=2" + depends_on: + - superlink + + supernode-2: + image: flwr/supernode:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9095 + - --isolation + - process + - --node-config + - "partition-id=1 num-partitions=2" + depends_on: + - superlink + + # uncomment to add another SuperNode + # + # supernode-3: + # image: flwr/supernode:${FLWR_VERSION:-1.15.2} + # command: + # - --insecure + # - --superlink + # - superlink:9092 + # - --clientappio-api-address + # - 0.0.0.0:9096 + # - --isolation + # - process + # - --node-config + # - "partition-id=1 num-partitions=2" + # depends_on: + # - superlink + + # create two ClientApp services + clientapp-1: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-1:9094 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-1 + + clientapp-2: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-2:9095 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-2 + # uncomment to add another ClientApp + # + # clientapp-3: + # build: + # context: ${PROJECT_DIR:-.} + # dockerfile_inline: | + # FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # # gcc is required for the fastai quickstart example + # USER root + # RUN apt-get update \ + # && apt-get -y --no-install-recommends install \ + # build-essential \ + # && rm -rf /var/lib/apt/lists/* + # USER app + + # WORKDIR /app + # COPY --chown=app:app pyproject.toml . + # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + # && python -m pip install -U --no-cache-dir . + + # ENTRYPOINT ["flwr-clientapp"] + # command: + # - --insecure + # - --clientappio-api-address + # - supernode-3:9096 + # deploy: + # resources: + # limits: + # cpus: "2" + # stop_signal: SIGINT + # depends_on: + # - supernode-3 diff --git a/flops/flower/application/__init__.py b/flops/flower/application/__init__.py new file mode 100644 index 00000000..71160e37 --- /dev/null +++ b/flops/flower/application/__init__.py @@ -0,0 +1 @@ +"""Flops: A Flower / PyTorch app.""" diff --git a/flops/flower/application/client_app.py b/flops/flower/application/client_app.py new file mode 100644 index 00000000..cc7dd891 --- /dev/null +++ b/flops/flower/application/client_app.py @@ -0,0 +1,55 @@ +"""Flops: A Flower / PyTorch app.""" + +import torch + +from flwr.client import ClientApp, NumPyClient +from flwr.common import Context +from application.task import Net, get_weights, load_data, set_weights, test, train + + +# Define Flower Client and client_fn +class FlowerClient(NumPyClient): + def __init__(self, net, trainloader, valloader, local_epochs): + self.net = net + self.trainloader = trainloader + self.valloader = valloader + self.local_epochs = local_epochs + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + self.net.to(self.device) + + def fit(self, parameters, config): + set_weights(self.net, parameters) + train_loss = train( + self.net, + self.trainloader, + self.local_epochs, + self.device, + ) + return ( + get_weights(self.net), + len(self.trainloader.dataset), + {"train_loss": train_loss}, + ) + + def evaluate(self, parameters, config): + set_weights(self.net, parameters) + loss, accuracy = test(self.net, self.valloader, self.device) + return loss, len(self.valloader.dataset), {"accuracy": accuracy} + + +def client_fn(context: Context): + # Load model and data + net = Net() + partition_id = context.node_config["partition-id"] + num_partitions = context.node_config["num-partitions"] + trainloader, valloader = load_data(partition_id, num_partitions) + local_epochs = context.run_config["local-epochs"] + + # Return Client instance + return FlowerClient(net, trainloader, valloader, local_epochs).to_client() + + +# Flower ClientApp +app = ClientApp( + client_fn, +) diff --git a/flops/flower/application/server_app.py b/flops/flower/application/server_app.py new file mode 100644 index 00000000..3fcfe0b7 --- /dev/null +++ b/flops/flower/application/server_app.py @@ -0,0 +1,31 @@ +"""Flops: A Flower / PyTorch app.""" + +from flwr.common import Context, ndarrays_to_parameters +from flwr.server import ServerApp, ServerAppComponents, ServerConfig +from flwr.server.strategy import FedAvg +from application.task import Net, get_weights + + +def server_fn(context: Context): + # Read from config + num_rounds = context.run_config["num-server-rounds"] + fraction_fit = context.run_config["fraction-fit"] + + # Initialize model parameters + ndarrays = get_weights(Net()) + parameters = ndarrays_to_parameters(ndarrays) + + # Define strategy + strategy = FedAvg( + fraction_fit=fraction_fit, + fraction_evaluate=1.0, + min_available_clients=2, + initial_parameters=parameters, + ) + config = ServerConfig(num_rounds=num_rounds) + + return ServerAppComponents(strategy=strategy, config=config) + + +# Create ServerApp +app = ServerApp(server_fn=server_fn) diff --git a/flops/flower/application/task.py b/flops/flower/application/task.py new file mode 100644 index 00000000..9cabd9b8 --- /dev/null +++ b/flops/flower/application/task.py @@ -0,0 +1,112 @@ +"""Flops: A Flower / PyTorch app.""" + +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from flwr_datasets import FederatedDataset +from flwr_datasets.partitioner import IidPartitioner +from torch.utils.data import DataLoader +from torchvision.transforms import Compose, Normalize, ToTensor + + +class Net(nn.Module): + """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" + + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + + +fds = None # Cache FederatedDataset + + +def load_data(partition_id: int, num_partitions: int): + """Load partition CIFAR10 data.""" + # Only initialize `FederatedDataset` once + global fds + if fds is None: + partitioner = IidPartitioner(num_partitions=num_partitions) + fds = FederatedDataset( + dataset="uoft-cs/cifar10", + partitioners={"train": partitioner}, + ) + partition = fds.load_partition(partition_id) + # Divide data on each node: 80% train, 20% test + partition_train_test = partition.train_test_split(test_size=0.2, seed=42) + pytorch_transforms = Compose( + [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + def apply_transforms(batch): + """Apply transforms to the partition from FederatedDataset.""" + batch["img"] = [pytorch_transforms(img) for img in batch["img"]] + return batch + + partition_train_test = partition_train_test.with_transform(apply_transforms) + trainloader = DataLoader(partition_train_test["train"], batch_size=32, shuffle=True) + testloader = DataLoader(partition_train_test["test"], batch_size=32) + return trainloader, testloader + + +def train(net, trainloader, epochs, device): + """Train the model on the training set.""" + net.to(device) # move model to GPU if available + criterion = torch.nn.CrossEntropyLoss().to(device) + optimizer = torch.optim.Adam(net.parameters(), lr=0.01) + net.train() + running_loss = 0.0 + for _ in range(epochs): + for batch in trainloader: + images = batch["img"] + labels = batch["label"] + optimizer.zero_grad() + loss = criterion(net(images.to(device)), labels.to(device)) + loss.backward() + optimizer.step() + running_loss += loss.item() + + avg_trainloss = running_loss / len(trainloader) + return avg_trainloss + + +def test(net, testloader, device): + """Validate the model on the test set.""" + net.to(device) + criterion = torch.nn.CrossEntropyLoss() + correct, loss = 0, 0.0 + with torch.no_grad(): + for batch in testloader: + images = batch["img"].to(device) + labels = batch["label"].to(device) + outputs = net(images) + loss += criterion(outputs, labels).item() + correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() + accuracy = correct / len(testloader.dataset) + loss = loss / len(testloader) + return loss, accuracy + + +def get_weights(net): + print('testing') + return [val.cpu().numpy() for _, val in net.state_dict().items()] + + +def set_weights(net, parameters): + params_dict = zip(net.state_dict().keys(), parameters) + state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) + net.load_state_dict(state_dict, strict=True) diff --git a/flops/flower/pyproject.toml b/flops/flower/pyproject.toml new file mode 100644 index 00000000..22e4deaa --- /dev/null +++ b/flops/flower/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "Flops" +version = "1.0.0" +description = "" +license = "Apache-2.0" +dependencies = [ + "flwr[simulation]>=1.15.2", + "flwr-datasets[vision]>=0.5.0", + "torch==2.5.1", + "torchvision==0.20.1", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.flwr.app] +publisher = "flower" + +[tool.flwr.app.components] +serverapp = "application.server_app:app" +clientapp = "application.client_app:app" + +[tool.flwr.app.config] +num-server-rounds = 3 +fraction-fit = 0.5 +local-epochs = 1 + +[tool.flwr.federations] +default = "local-simulation" + +[tool.flwr.federations.local-simulation] +options.num-supernodes = 10 + +[tool.flwr.federations.local-deployment] +address = "127.0.0.1:9093" +insecure = true \ No newline at end of file From abf9902f1161feb2b46d82ee3ec17e168d44ecc5 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Mon, 10 Mar 2025 01:25:18 +0530 Subject: [PATCH 02/18] running flower with k8s --- aiopslab/paths.py | 1 + aiopslab/service/apps/flower.py | 69 +++++++++ aiopslab/service/metadata/flower.json | 10 ++ flops/certs.yml | 98 ------------- flops/compose.yml | 185 ------------------------- flops/flower/application/__init__.py | 1 - flops/flower/application/client_app.py | 55 -------- flops/flower/application/server_app.py | 31 ----- flops/flower/application/task.py | 112 --------------- flops/flower/pyproject.toml | 40 ------ 10 files changed, 80 insertions(+), 522 deletions(-) create mode 100644 aiopslab/service/apps/flower.py create mode 100644 aiopslab/service/metadata/flower.json delete mode 100644 flops/certs.yml delete mode 100644 flops/compose.yml delete mode 100644 flops/flower/application/__init__.py delete mode 100644 flops/flower/application/client_app.py delete mode 100644 flops/flower/application/server_app.py delete mode 100644 flops/flower/application/task.py delete mode 100644 flops/flower/pyproject.toml diff --git a/aiopslab/paths.py b/aiopslab/paths.py index 6aad9b2a..2391bc41 100644 --- a/aiopslab/paths.py +++ b/aiopslab/paths.py @@ -34,3 +34,4 @@ ASTRONOMY_SHOP_METADATA = BASE_DIR / "service" / "metadata" / "astronomy-shop.json" TIDB_METADATA = BASE_DIR / "service" / "metadata" / "tidb-with-operator.json" FLIGHT_TICKET_METADATA = BASE_DIR / "service" / "metadata" / "flight-ticket.json" +FLOWER_METADATA = BASE_DIR / "service" / "metadata" / "flower.json" \ No newline at end of file diff --git a/aiopslab/service/apps/flower.py b/aiopslab/service/apps/flower.py new file mode 100644 index 00000000..3280c229 --- /dev/null +++ b/aiopslab/service/apps/flower.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../'))) + +import time +from aiopslab.service.kubectl import KubeCtl +from aiopslab.service.apps.base import Application +from aiopslab.paths import FAULT_SCRIPTS, FLOWER_METADATA + + +class Flower(Application): + def __init__(self): + super().__init__(FLOWER_METADATA) + self.kubectl = KubeCtl() + self.script_dir = FAULT_SCRIPTS + self.helm_deploy = False + + self.load_app_json() + self.create_namespace() + + def load_app_json(self): + super().load_app_json() + metadata = self.get_app_json() + self.frontend_service = None + self.frontend_port = None + + def deploy(self): + """Deploy the Kubernetes configurations.""" + print(f"Deploying Kubernetes configurations in namespace: {self.namespace}") + self.kubectl.apply_configs(self.namespace, self.k8s_deploy_path) + self.kubectl.wait_for_ready(self.namespace) + + def delete(self): + """Delete the configmap.""" + self.kubectl.delete_configs(self.namespace, self.k8s_deploy_path) + + def cleanup(self): + """Delete the entire namespace for the flower application.""" + self.kubectl.delete_namespace(self.namespace) + time.sleep(10) + pvs = self.kubectl.exec_command( + "kubectl get pv --no-headers | grep 'test-flower' | awk '{print $1}'" + ).splitlines() + + for pv in pvs: + # Check if the PV is in a 'Terminating' state and remove the finalizers if necessary + self._remove_pv_finalizers(pv) + delete_command = f"kubectl delete pv {pv}" + delete_result = self.kubectl.exec_command(delete_command) + print(f"Deleted PersistentVolume {pv}: {delete_result.strip()}") + time.sleep(5) + + def _remove_pv_finalizers(self, pv_name: str): + """Remove finalizers from the PersistentVolume to prevent it from being stuck in a 'Terminating' state.""" + # Patch the PersistentVolume to remove finalizers if it is stuck + patch_command = ( + f'kubectl patch pv {pv_name} -p \'{{"metadata":{{"finalizers":null}}}}\'' + ) + _ = self.kubectl.exec_command(patch_command) + + +if __name__ == "__main__": + flower = Flower() + flower.deploy() + flower.delete() + flower.cleanup() \ No newline at end of file diff --git a/aiopslab/service/metadata/flower.json b/aiopslab/service/metadata/flower.json new file mode 100644 index 00000000..15b77674 --- /dev/null +++ b/aiopslab/service/metadata/flower.json @@ -0,0 +1,10 @@ +{ + "Name": "Flower", + "Namespace": "test-flower", + "Desc": "A federated learning application to train models on edge devices and aggregate them on a central server.", + "Supported Operations": [ + "Train models on edge devices", + "Aggregate models on a central server" + ], + "K8S Deploy Path": "flower/kubernetes" +} \ No newline at end of file diff --git a/flops/certs.yml b/flops/certs.yml deleted file mode 100644 index 863c702f..00000000 --- a/flops/certs.yml +++ /dev/null @@ -1,98 +0,0 @@ -services: - gen-certs: - build: - context: . - pull: true - dockerfile_inline: | - FROM ubuntu:latest - - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - openssl - - WORKDIR /app/script - - ARG SUPERLINK_IP=127.0.0.1 - - COPY <<-EOF superlink-certificate.conf - [req] - default_bits = 4096 - prompt = no - default_md = sha256 - req_extensions = req_ext - distinguished_name = dn - - [dn] - C = US - O = Flower - CN = localhost - - [req_ext] - subjectAltName = @alt_names - - [alt_names] - DNS.0 = superlink - IP.1 = ::1 - IP.2 = $${SUPERLINK_IP} - EOF - - COPY --chmod=744 <<-'EOF' generate.sh - #!/bin/bash - # This script will generate all certificates if ca.crt does not exist - - set -e - cd "$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ - - CA_PASSWORD=notsafe - - # Generate directories if not exists - - generate () { - mkdir -p "$$1" - - if [ -f ""$$1"/ca.crt" ]; then - echo "Skipping certificate generation as they already exist." - return 0 - fi - - # Generate the root certificate authority key and certificate based on key - openssl genrsa -out "$$1"/ca.key 4096 - openssl req \ - -new \ - -x509 \ - -key "$$1"/ca.key \ - -sha256 \ - -subj "/C=DE/ST=HH/O=CA, Inc." \ - -days 365 -out "$$1"/ca.crt - - # Generate a new private key for the server - openssl genrsa -out "$$1"/server.key 4096 - - # Create a signing CSR - openssl req \ - -new \ - -key "$$1"/server.key \ - -out "$$1"/server.csr \ - -config ./script/"$$2" - - # Generate a certificate for the server - openssl x509 \ - -req \ - -in "$$1"/server.csr \ - -CA "$$1"/ca.crt \ - -CAkey "$$1"/ca.key \ - -CAcreateserial \ - -out "$$1"/server.pem \ - -days 365 \ - -sha256 \ - -extfile ./script/"$$2" \ - -extensions req_ext - } - generate superlink-certificates superlink-certificate.conf - EOF - - WORKDIR /app - - ENTRYPOINT ["./script/generate.sh"] - volumes: - - ./superlink-certificates/:/app/superlink-certificates/:rw diff --git a/flops/compose.yml b/flops/compose.yml deleted file mode 100644 index 9a19788e..00000000 --- a/flops/compose.yml +++ /dev/null @@ -1,185 +0,0 @@ -services: - # create a SuperLink service - superlink: - image: flwr/superlink:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --isolation - - process - ports: - - 9093:9093 - - # create a ServerApp service - serverapp: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/serverapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-serverapp"] - command: - - --insecure - - --serverappio-api-address - - superlink:9091 - restart: on-failure:3 - depends_on: - - superlink - - # create two SuperNode services with different node configs - supernode-1: - image: flwr/supernode:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9094 - - --isolation - - process - - --node-config - - "partition-id=0 num-partitions=2" - depends_on: - - superlink - - supernode-2: - image: flwr/supernode:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9095 - - --isolation - - process - - --node-config - - "partition-id=1 num-partitions=2" - depends_on: - - superlink - - # uncomment to add another SuperNode - # - # supernode-3: - # image: flwr/supernode:${FLWR_VERSION:-1.15.2} - # command: - # - --insecure - # - --superlink - # - superlink:9092 - # - --clientappio-api-address - # - 0.0.0.0:9096 - # - --isolation - # - process - # - --node-config - # - "partition-id=1 num-partitions=2" - # depends_on: - # - superlink - - # create two ClientApp services - clientapp-1: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-1:9094 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-1 - - clientapp-2: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-2:9095 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-2 - # uncomment to add another ClientApp - # - # clientapp-3: - # build: - # context: ${PROJECT_DIR:-.} - # dockerfile_inline: | - # FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # # gcc is required for the fastai quickstart example - # USER root - # RUN apt-get update \ - # && apt-get -y --no-install-recommends install \ - # build-essential \ - # && rm -rf /var/lib/apt/lists/* - # USER app - - # WORKDIR /app - # COPY --chown=app:app pyproject.toml . - # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - # && python -m pip install -U --no-cache-dir . - - # ENTRYPOINT ["flwr-clientapp"] - # command: - # - --insecure - # - --clientappio-api-address - # - supernode-3:9096 - # deploy: - # resources: - # limits: - # cpus: "2" - # stop_signal: SIGINT - # depends_on: - # - supernode-3 diff --git a/flops/flower/application/__init__.py b/flops/flower/application/__init__.py deleted file mode 100644 index 71160e37..00000000 --- a/flops/flower/application/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" diff --git a/flops/flower/application/client_app.py b/flops/flower/application/client_app.py deleted file mode 100644 index cc7dd891..00000000 --- a/flops/flower/application/client_app.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -import torch - -from flwr.client import ClientApp, NumPyClient -from flwr.common import Context -from application.task import Net, get_weights, load_data, set_weights, test, train - - -# Define Flower Client and client_fn -class FlowerClient(NumPyClient): - def __init__(self, net, trainloader, valloader, local_epochs): - self.net = net - self.trainloader = trainloader - self.valloader = valloader - self.local_epochs = local_epochs - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.net.to(self.device) - - def fit(self, parameters, config): - set_weights(self.net, parameters) - train_loss = train( - self.net, - self.trainloader, - self.local_epochs, - self.device, - ) - return ( - get_weights(self.net), - len(self.trainloader.dataset), - {"train_loss": train_loss}, - ) - - def evaluate(self, parameters, config): - set_weights(self.net, parameters) - loss, accuracy = test(self.net, self.valloader, self.device) - return loss, len(self.valloader.dataset), {"accuracy": accuracy} - - -def client_fn(context: Context): - # Load model and data - net = Net() - partition_id = context.node_config["partition-id"] - num_partitions = context.node_config["num-partitions"] - trainloader, valloader = load_data(partition_id, num_partitions) - local_epochs = context.run_config["local-epochs"] - - # Return Client instance - return FlowerClient(net, trainloader, valloader, local_epochs).to_client() - - -# Flower ClientApp -app = ClientApp( - client_fn, -) diff --git a/flops/flower/application/server_app.py b/flops/flower/application/server_app.py deleted file mode 100644 index 3fcfe0b7..00000000 --- a/flops/flower/application/server_app.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -from flwr.common import Context, ndarrays_to_parameters -from flwr.server import ServerApp, ServerAppComponents, ServerConfig -from flwr.server.strategy import FedAvg -from application.task import Net, get_weights - - -def server_fn(context: Context): - # Read from config - num_rounds = context.run_config["num-server-rounds"] - fraction_fit = context.run_config["fraction-fit"] - - # Initialize model parameters - ndarrays = get_weights(Net()) - parameters = ndarrays_to_parameters(ndarrays) - - # Define strategy - strategy = FedAvg( - fraction_fit=fraction_fit, - fraction_evaluate=1.0, - min_available_clients=2, - initial_parameters=parameters, - ) - config = ServerConfig(num_rounds=num_rounds) - - return ServerAppComponents(strategy=strategy, config=config) - - -# Create ServerApp -app = ServerApp(server_fn=server_fn) diff --git a/flops/flower/application/task.py b/flops/flower/application/task.py deleted file mode 100644 index 9cabd9b8..00000000 --- a/flops/flower/application/task.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F -from flwr_datasets import FederatedDataset -from flwr_datasets.partitioner import IidPartitioner -from torch.utils.data import DataLoader -from torchvision.transforms import Compose, Normalize, ToTensor - - -class Net(nn.Module): - """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" - - def __init__(self): - super(Net, self).__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(-1, 16 * 5 * 5) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - return self.fc3(x) - - -fds = None # Cache FederatedDataset - - -def load_data(partition_id: int, num_partitions: int): - """Load partition CIFAR10 data.""" - # Only initialize `FederatedDataset` once - global fds - if fds is None: - partitioner = IidPartitioner(num_partitions=num_partitions) - fds = FederatedDataset( - dataset="uoft-cs/cifar10", - partitioners={"train": partitioner}, - ) - partition = fds.load_partition(partition_id) - # Divide data on each node: 80% train, 20% test - partition_train_test = partition.train_test_split(test_size=0.2, seed=42) - pytorch_transforms = Compose( - [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - - def apply_transforms(batch): - """Apply transforms to the partition from FederatedDataset.""" - batch["img"] = [pytorch_transforms(img) for img in batch["img"]] - return batch - - partition_train_test = partition_train_test.with_transform(apply_transforms) - trainloader = DataLoader(partition_train_test["train"], batch_size=32, shuffle=True) - testloader = DataLoader(partition_train_test["test"], batch_size=32) - return trainloader, testloader - - -def train(net, trainloader, epochs, device): - """Train the model on the training set.""" - net.to(device) # move model to GPU if available - criterion = torch.nn.CrossEntropyLoss().to(device) - optimizer = torch.optim.Adam(net.parameters(), lr=0.01) - net.train() - running_loss = 0.0 - for _ in range(epochs): - for batch in trainloader: - images = batch["img"] - labels = batch["label"] - optimizer.zero_grad() - loss = criterion(net(images.to(device)), labels.to(device)) - loss.backward() - optimizer.step() - running_loss += loss.item() - - avg_trainloss = running_loss / len(trainloader) - return avg_trainloss - - -def test(net, testloader, device): - """Validate the model on the test set.""" - net.to(device) - criterion = torch.nn.CrossEntropyLoss() - correct, loss = 0, 0.0 - with torch.no_grad(): - for batch in testloader: - images = batch["img"].to(device) - labels = batch["label"].to(device) - outputs = net(images) - loss += criterion(outputs, labels).item() - correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() - accuracy = correct / len(testloader.dataset) - loss = loss / len(testloader) - return loss, accuracy - - -def get_weights(net): - print('testing') - return [val.cpu().numpy() for _, val in net.state_dict().items()] - - -def set_weights(net, parameters): - params_dict = zip(net.state_dict().keys(), parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - net.load_state_dict(state_dict, strict=True) diff --git a/flops/flower/pyproject.toml b/flops/flower/pyproject.toml deleted file mode 100644 index 22e4deaa..00000000 --- a/flops/flower/pyproject.toml +++ /dev/null @@ -1,40 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "Flops" -version = "1.0.0" -description = "" -license = "Apache-2.0" -dependencies = [ - "flwr[simulation]>=1.15.2", - "flwr-datasets[vision]>=0.5.0", - "torch==2.5.1", - "torchvision==0.20.1", -] - -[tool.hatch.build.targets.wheel] -packages = ["."] - -[tool.flwr.app] -publisher = "flower" - -[tool.flwr.app.components] -serverapp = "application.server_app:app" -clientapp = "application.client_app:app" - -[tool.flwr.app.config] -num-server-rounds = 3 -fraction-fit = 0.5 -local-epochs = 1 - -[tool.flwr.federations] -default = "local-simulation" - -[tool.flwr.federations.local-simulation] -options.num-supernodes = 10 - -[tool.flwr.federations.local-deployment] -address = "127.0.0.1:9093" -insecure = true \ No newline at end of file From 3702ca376ae8adf514a27573b75319daa71808de Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Mon, 10 Mar 2025 01:33:31 +0530 Subject: [PATCH 03/18] added flower --- aiopslab-applications | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiopslab-applications b/aiopslab-applications index c92ca583..f0eca436 160000 --- a/aiopslab-applications +++ b/aiopslab-applications @@ -1 +1 @@ -Subproject commit c92ca583702849ef4555d74f20d3528c2bbe8ee0 +Subproject commit f0eca4366756d5508f8a2629f0f9fcc5e546b0e2 From bc0b2476fa182ab414c9fdab39d2f5deb1aa96d1 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Mon, 10 Mar 2025 13:33:36 +0530 Subject: [PATCH 04/18] flower workload --- aiopslab-applications | 2 +- .../problems/flower/pyproject.toml | 40 +++++++ .../problems/flower/train/__init__.py | 1 + .../problems/flower/train/client_app.py | 55 +++++++++ .../problems/flower/train/server_app.py | 31 +++++ .../problems/flower/train/task.py | 112 ++++++++++++++++++ 6 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 aiopslab/orchestrator/problems/flower/pyproject.toml create mode 100644 aiopslab/orchestrator/problems/flower/train/__init__.py create mode 100644 aiopslab/orchestrator/problems/flower/train/client_app.py create mode 100644 aiopslab/orchestrator/problems/flower/train/server_app.py create mode 100644 aiopslab/orchestrator/problems/flower/train/task.py diff --git a/aiopslab-applications b/aiopslab-applications index f0eca436..553da3f1 160000 --- a/aiopslab-applications +++ b/aiopslab-applications @@ -1 +1 @@ -Subproject commit f0eca4366756d5508f8a2629f0f9fcc5e546b0e2 +Subproject commit 553da3f1bfcabfe597da8c58398a888b9331cd3b diff --git a/aiopslab/orchestrator/problems/flower/pyproject.toml b/aiopslab/orchestrator/problems/flower/pyproject.toml new file mode 100644 index 00000000..a945fb91 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "Flops" +version = "1.0.0" +description = "" +license = "Apache-2.0" +dependencies = [ + "flwr[simulation]>=1.15.2", + "flwr-datasets[vision]>=0.5.0", + "torch==2.5.1", + "torchvision==0.20.1", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.flwr.app] +publisher = "flower" + +[tool.flwr.app.components] +serverapp = "train.server_app:app" +clientapp = "train.client_app:app" + +[tool.flwr.app.config] +num-server-rounds = 3 +fraction-fit = 0.5 +local-epochs = 1 + +[tool.flwr.federations] +default = "local-simulation" + +[tool.flwr.federations.local-simulation] +options.num-supernodes = 10 + +[tool.flwr.federations.local-deployment] +address = "127.0.0.1:9093" +insecure = true \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/train/__init__.py b/aiopslab/orchestrator/problems/flower/train/__init__.py new file mode 100644 index 00000000..71160e37 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/train/__init__.py @@ -0,0 +1 @@ +"""Flops: A Flower / PyTorch app.""" diff --git a/aiopslab/orchestrator/problems/flower/train/client_app.py b/aiopslab/orchestrator/problems/flower/train/client_app.py new file mode 100644 index 00000000..10102508 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/train/client_app.py @@ -0,0 +1,55 @@ +"""Flops: A Flower / PyTorch app.""" + +import torch + +from flwr.client import ClientApp, NumPyClient +from flwr.common import Context +from train.task import Net, get_weights, load_data, set_weights, test, train + + +# Define Flower Client and client_fn +class FlowerClient(NumPyClient): + def __init__(self, net, trainloader, valloader, local_epochs): + self.net = net + self.trainloader = trainloader + self.valloader = valloader + self.local_epochs = local_epochs + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + self.net.to(self.device) + + def fit(self, parameters, config): + set_weights(self.net, parameters) + train_loss = train( + self.net, + self.trainloader, + self.local_epochs, + self.device, + ) + return ( + get_weights(self.net), + len(self.trainloader.dataset), + {"train_loss": train_loss}, + ) + + def evaluate(self, parameters, config): + set_weights(self.net, parameters) + loss, accuracy = test(self.net, self.valloader, self.device) + return loss, len(self.valloader.dataset), {"accuracy": accuracy} + + +def client_fn(context: Context): + # Load model and data + net = Net() + partition_id = context.node_config["partition-id"] + num_partitions = context.node_config["num-partitions"] + trainloader, valloader = load_data(partition_id, num_partitions) + local_epochs = context.run_config["local-epochs"] + + # Return Client instance + return FlowerClient(net, trainloader, valloader, local_epochs).to_client() + + +# Flower ClientApp +app = ClientApp( + client_fn, +) diff --git a/aiopslab/orchestrator/problems/flower/train/server_app.py b/aiopslab/orchestrator/problems/flower/train/server_app.py new file mode 100644 index 00000000..43c74bcb --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/train/server_app.py @@ -0,0 +1,31 @@ +"""Flops: A Flower / PyTorch app.""" + +from flwr.common import Context, ndarrays_to_parameters +from flwr.server import ServerApp, ServerAppComponents, ServerConfig +from flwr.server.strategy import FedAvg +from train.task import Net, get_weights + + +def server_fn(context: Context): + # Read from config + num_rounds = context.run_config["num-server-rounds"] + fraction_fit = context.run_config["fraction-fit"] + + # Initialize model parameters + ndarrays = get_weights(Net()) + parameters = ndarrays_to_parameters(ndarrays) + + # Define strategy + strategy = FedAvg( + fraction_fit=fraction_fit, + fraction_evaluate=1.0, + min_available_clients=2, + initial_parameters=parameters, + ) + config = ServerConfig(num_rounds=num_rounds) + + return ServerAppComponents(strategy=strategy, config=config) + + +# Create ServerApp +app = ServerApp(server_fn=server_fn) diff --git a/aiopslab/orchestrator/problems/flower/train/task.py b/aiopslab/orchestrator/problems/flower/train/task.py new file mode 100644 index 00000000..9cabd9b8 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/train/task.py @@ -0,0 +1,112 @@ +"""Flops: A Flower / PyTorch app.""" + +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from flwr_datasets import FederatedDataset +from flwr_datasets.partitioner import IidPartitioner +from torch.utils.data import DataLoader +from torchvision.transforms import Compose, Normalize, ToTensor + + +class Net(nn.Module): + """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" + + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + + +fds = None # Cache FederatedDataset + + +def load_data(partition_id: int, num_partitions: int): + """Load partition CIFAR10 data.""" + # Only initialize `FederatedDataset` once + global fds + if fds is None: + partitioner = IidPartitioner(num_partitions=num_partitions) + fds = FederatedDataset( + dataset="uoft-cs/cifar10", + partitioners={"train": partitioner}, + ) + partition = fds.load_partition(partition_id) + # Divide data on each node: 80% train, 20% test + partition_train_test = partition.train_test_split(test_size=0.2, seed=42) + pytorch_transforms = Compose( + [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + def apply_transforms(batch): + """Apply transforms to the partition from FederatedDataset.""" + batch["img"] = [pytorch_transforms(img) for img in batch["img"]] + return batch + + partition_train_test = partition_train_test.with_transform(apply_transforms) + trainloader = DataLoader(partition_train_test["train"], batch_size=32, shuffle=True) + testloader = DataLoader(partition_train_test["test"], batch_size=32) + return trainloader, testloader + + +def train(net, trainloader, epochs, device): + """Train the model on the training set.""" + net.to(device) # move model to GPU if available + criterion = torch.nn.CrossEntropyLoss().to(device) + optimizer = torch.optim.Adam(net.parameters(), lr=0.01) + net.train() + running_loss = 0.0 + for _ in range(epochs): + for batch in trainloader: + images = batch["img"] + labels = batch["label"] + optimizer.zero_grad() + loss = criterion(net(images.to(device)), labels.to(device)) + loss.backward() + optimizer.step() + running_loss += loss.item() + + avg_trainloss = running_loss / len(trainloader) + return avg_trainloss + + +def test(net, testloader, device): + """Validate the model on the test set.""" + net.to(device) + criterion = torch.nn.CrossEntropyLoss() + correct, loss = 0, 0.0 + with torch.no_grad(): + for batch in testloader: + images = batch["img"].to(device) + labels = batch["label"].to(device) + outputs = net(images) + loss += criterion(outputs, labels).item() + correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() + accuracy = correct / len(testloader.dataset) + loss = loss / len(testloader) + return loss, accuracy + + +def get_weights(net): + print('testing') + return [val.cpu().numpy() for _, val in net.state_dict().items()] + + +def set_weights(net, parameters): + params_dict = zip(net.state_dict().keys(), parameters) + state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) + net.load_state_dict(state_dict, strict=True) From 615aaa17bfa43a65d1b4d94ef74e477583eb932c Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Tue, 11 Mar 2025 03:13:16 +0530 Subject: [PATCH 05/18] added flower docker and k8s files --- flower/compose.yml | 186 ++++++++++++++++++ flower/config/pyproject.toml | 36 ++++ .../clientapp/clientapp-1-deployment.yaml | 36 ++++ .../clientapp/clientapp-1-service.yaml | 16 ++ .../clientapp/clientapp-2-deployment.yaml | 36 ++++ .../clientapp/clientapp-2-service.yaml | 16 ++ .../kubernetes/serverapp/serverapp-pod.yaml | 21 ++ .../serverapp/serverapp-service.yaml | 16 ++ .../superlink/superlink-deployment.yaml | 33 ++++ .../superlink/superlink-service.yaml | 16 ++ .../supernode/supernode-1-deployment.yaml | 39 ++++ .../supernode/supernode-1-service.yaml | 16 ++ .../supernode/supernode-2-deployment.yaml | 39 ++++ .../supernode/supernode-2-service.yaml | 16 ++ 14 files changed, 522 insertions(+) create mode 100644 flower/compose.yml create mode 100644 flower/config/pyproject.toml create mode 100644 flower/kubernetes/clientapp/clientapp-1-deployment.yaml create mode 100644 flower/kubernetes/clientapp/clientapp-1-service.yaml create mode 100644 flower/kubernetes/clientapp/clientapp-2-deployment.yaml create mode 100644 flower/kubernetes/clientapp/clientapp-2-service.yaml create mode 100644 flower/kubernetes/serverapp/serverapp-pod.yaml create mode 100644 flower/kubernetes/serverapp/serverapp-service.yaml create mode 100644 flower/kubernetes/superlink/superlink-deployment.yaml create mode 100644 flower/kubernetes/superlink/superlink-service.yaml create mode 100644 flower/kubernetes/supernode/supernode-1-deployment.yaml create mode 100644 flower/kubernetes/supernode/supernode-1-service.yaml create mode 100644 flower/kubernetes/supernode/supernode-2-deployment.yaml create mode 100644 flower/kubernetes/supernode/supernode-2-service.yaml diff --git a/flower/compose.yml b/flower/compose.yml new file mode 100644 index 00000000..8753ddbc --- /dev/null +++ b/flower/compose.yml @@ -0,0 +1,186 @@ +services: + # create a SuperLink service + superlink: + image: flwr/superlink:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --isolation + - process + ports: + - 9093:9093 + + # create a ServerApp service + serverapp: + build: + context: config + dockerfile_inline: | + FROM flwr/serverapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-serverapp"] + command: + - --insecure + - --serverappio-api-address + - superlink:9091 + restart: on-failure + depends_on: + - superlink + + # create two SuperNode services with different node configs + supernode-1: + image: flwr/supernode:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9094 + - --isolation + - process + - --node-config + - "partition-id=0 num-partitions=2" + depends_on: + - superlink + + supernode-2: + image: flwr/supernode:${FLWR_VERSION:-1.15.2} + command: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9095 + - --isolation + - process + - --node-config + - "partition-id=1 num-partitions=2" + depends_on: + - superlink + + # uncomment to add another SuperNode + # + # supernode-3: + # image: flwr/supernode:${FLWR_VERSION:-1.15.2} + # command: + # - --insecure + # - --superlink + # - superlink:9092 + # - --clientappio-api-address + # - 0.0.0.0:9096 + # - --isolation + # - process + # - --node-config + # - "partition-id=1 num-partitions=2" + # depends_on: + # - superlink + + # create two ClientApp services + clientapp-1: + build: + context: config + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-1:9094 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-1 + + clientapp-2: + build: + context: config + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # gcc is required for the fastai quickstart example + USER root + RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + USER app + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-2:9095 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-2 + + # uncomment to add another ClientApp + # + # clientapp-3: + # build: + # context: config + # dockerfile_inline: | + # FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} + + # # gcc is required for the fastai quickstart example + # USER root + # RUN apt-get update \ + # && apt-get -y --no-install-recommends install \ + # build-essential \ + # && rm -rf /var/lib/apt/lists/* + # USER app + + # WORKDIR /app + # COPY --chown=app:app pyproject.toml . + # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + # && python -m pip install -U --no-cache-dir . + + # ENTRYPOINT ["flwr-clientapp"] + # command: + # - --insecure + # - --clientappio-api-address + # - supernode-3:9096 + # deploy: + # resources: + # limits: + # cpus: "2" + # stop_signal: SIGINT + # depends_on: + # - supernode-3 diff --git a/flower/config/pyproject.toml b/flower/config/pyproject.toml new file mode 100644 index 00000000..b17fa6da --- /dev/null +++ b/flower/config/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "Flops" +version = "1.0.0" +description = "" +license = "Apache-2.0" +dependencies = [ + "flwr[simulation]>=1.15.2", + "flwr-datasets[vision]>=0.5.0", + "torch==2.5.1", + "torchvision==0.20.1", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.flwr.app] +publisher = "flower" + +[tool.flwr.app.components] +serverapp = "train.server_app:app" +clientapp = "train.client_app:app" + +[tool.flwr.app.config] +num-server-rounds = 3 +fraction-fit = 0.5 +local-epochs = 1 + +[tool.flwr.federations] +default = "local-simulation" + +[tool.flwr.federations.local-simulation] +options.num-supernodes = 10 \ No newline at end of file diff --git a/flower/kubernetes/clientapp/clientapp-1-deployment.yaml b/flower/kubernetes/clientapp/clientapp-1-deployment.yaml new file mode 100644 index 00000000..4759f0a4 --- /dev/null +++ b/flower/kubernetes/clientapp/clientapp-1-deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-1 + name: clientapp-1 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: clientapp-1 + template: + metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-1 + spec: + containers: + - args: + - --insecure + - --clientappio-api-address + - supernode-1:9094 + image: adityapgupta/flower-client-1:latest + name: clientapp-1 + ports: + - containerPort: 9094 + protocol: TCP + resources: + limits: + cpu: "2" + restartPolicy: Always diff --git a/flower/kubernetes/clientapp/clientapp-1-service.yaml b/flower/kubernetes/clientapp/clientapp-1-service.yaml new file mode 100644 index 00000000..eb26aeaf --- /dev/null +++ b/flower/kubernetes/clientapp/clientapp-1-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-1 + name: clientapp-1 +spec: + ports: + - name: "9094" + port: 9094 + targetPort: 9094 + selector: + io.kompose.service: clientapp-1 diff --git a/flower/kubernetes/clientapp/clientapp-2-deployment.yaml b/flower/kubernetes/clientapp/clientapp-2-deployment.yaml new file mode 100644 index 00000000..09abaabc --- /dev/null +++ b/flower/kubernetes/clientapp/clientapp-2-deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-2 + name: clientapp-2 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: clientapp-2 + template: + metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-2 + spec: + containers: + - args: + - --insecure + - --clientappio-api-address + - supernode-2:9095 + image: adityapgupta/flower-client-2:latest + name: clientapp-2 + ports: + - containerPort: 9095 + protocol: TCP + resources: + limits: + cpu: "2" + restartPolicy: Always diff --git a/flower/kubernetes/clientapp/clientapp-2-service.yaml b/flower/kubernetes/clientapp/clientapp-2-service.yaml new file mode 100644 index 00000000..07c40063 --- /dev/null +++ b/flower/kubernetes/clientapp/clientapp-2-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: clientapp-2 + name: clientapp-2 +spec: + ports: + - name: "9095" + port: 9095 + targetPort: 9095 + selector: + io.kompose.service: clientapp-2 diff --git a/flower/kubernetes/serverapp/serverapp-pod.yaml b/flower/kubernetes/serverapp/serverapp-pod.yaml new file mode 100644 index 00000000..08edf1cf --- /dev/null +++ b/flower/kubernetes/serverapp/serverapp-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: serverapp + name: serverapp +spec: + containers: + - args: + - --insecure + - --serverappio-api-address + - superlink:9091 + image: adityapgupta/flower-server:latest + name: serverapp + ports: + - containerPort: 9091 + protocol: TCP + restartPolicy: OnFailure diff --git a/flower/kubernetes/serverapp/serverapp-service.yaml b/flower/kubernetes/serverapp/serverapp-service.yaml new file mode 100644 index 00000000..bf618e47 --- /dev/null +++ b/flower/kubernetes/serverapp/serverapp-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: serverapp + name: serverapp +spec: + ports: + - name: "9091" + port: 9091 + targetPort: 9091 + selector: + io.kompose.service: serverapp diff --git a/flower/kubernetes/superlink/superlink-deployment.yaml b/flower/kubernetes/superlink/superlink-deployment.yaml new file mode 100644 index 00000000..0f77f4ef --- /dev/null +++ b/flower/kubernetes/superlink/superlink-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: superlink + name: superlink +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: superlink + template: + metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: superlink + spec: + containers: + - args: + - --insecure + - --isolation + - process + image: flwr/superlink:1.15.2 + name: superlink + ports: + - containerPort: 9093 + protocol: TCP + restartPolicy: Always diff --git a/flower/kubernetes/superlink/superlink-service.yaml b/flower/kubernetes/superlink/superlink-service.yaml new file mode 100644 index 00000000..114c3ea5 --- /dev/null +++ b/flower/kubernetes/superlink/superlink-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: superlink + name: superlink +spec: + ports: + - name: "9093" + port: 9093 + targetPort: 9093 + selector: + io.kompose.service: superlink diff --git a/flower/kubernetes/supernode/supernode-1-deployment.yaml b/flower/kubernetes/supernode/supernode-1-deployment.yaml new file mode 100644 index 00000000..c1957f9e --- /dev/null +++ b/flower/kubernetes/supernode/supernode-1-deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-1 + name: supernode-1 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: supernode-1 + template: + metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-1 + spec: + containers: + - args: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9094 + - --isolation + - process + - --node-config + - partition-id=0 num-partitions=2 + image: flwr/supernode:1.15.2 + name: supernode-1 + ports: + - containerPort: 9094 + protocol: TCP + restartPolicy: Always diff --git a/flower/kubernetes/supernode/supernode-1-service.yaml b/flower/kubernetes/supernode/supernode-1-service.yaml new file mode 100644 index 00000000..aa744e3f --- /dev/null +++ b/flower/kubernetes/supernode/supernode-1-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-1 + name: supernode-1 +spec: + ports: + - name: "9094" + port: 9094 + targetPort: 9094 + selector: + io.kompose.service: supernode-1 diff --git a/flower/kubernetes/supernode/supernode-2-deployment.yaml b/flower/kubernetes/supernode/supernode-2-deployment.yaml new file mode 100644 index 00000000..5ea2a4d3 --- /dev/null +++ b/flower/kubernetes/supernode/supernode-2-deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-2 + name: supernode-2 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: supernode-2 + template: + metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-2 + spec: + containers: + - args: + - --insecure + - --superlink + - superlink:9092 + - --clientappio-api-address + - 0.0.0.0:9095 + - --isolation + - process + - --node-config + - partition-id=1 num-partitions=2 + image: flwr/supernode:1.15.2 + name: supernode-2 + ports: + - containerPort: 9095 + protocol: TCP + restartPolicy: Always diff --git a/flower/kubernetes/supernode/supernode-2-service.yaml b/flower/kubernetes/supernode/supernode-2-service.yaml new file mode 100644 index 00000000..2de7487c --- /dev/null +++ b/flower/kubernetes/supernode/supernode-2-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: kompose convert + kompose.version: 1.35.0 (9532ceef3) + labels: + io.kompose.service: supernode-2 + name: supernode-2 +spec: + ports: + - name: "9095" + port: 9095 + targetPort: 9095 + selector: + io.kompose.service: supernode-2 From 2d3b410c364cad6ca9073f29e73728419c0018b3 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Tue, 11 Mar 2025 12:45:52 +0530 Subject: [PATCH 06/18] updated submodule --- aiopslab-applications | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiopslab-applications b/aiopslab-applications index 553da3f1..5221ef96 160000 --- a/aiopslab-applications +++ b/aiopslab-applications @@ -1 +1 @@ -Subproject commit 553da3f1bfcabfe597da8c58398a888b9331cd3b +Subproject commit 5221ef962879546bb3c977c8a256ce117697b9f2 From b91af01b3a6633de2ed735fb558bf2bfca707054 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Wed, 12 Mar 2025 18:42:52 +0530 Subject: [PATCH 07/18] testing edits --- .../orchestrator/problems/flower/__init__.py | 1 + .../problems/flower/flower_test.py | 9 ++ .../problems/flower/pyproject.toml | 2 +- aiopslab/orchestrator/problems/registry.py | 3 + aiopslab/service/apps/flower.py | 4 +- distributed/.gitignore | 2 + distributed/certs.yml | 5 + distributed/client/compose.yml | 130 ++++++++++++++++++ distributed/server/compose.yml | 49 +++++++ .../kubernetes/serverapp/serverapp-pod.yaml | 2 + .../superlink/superlink-deployment.yaml | 2 + 11 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 aiopslab/orchestrator/problems/flower/__init__.py create mode 100644 aiopslab/orchestrator/problems/flower/flower_test.py create mode 100644 distributed/.gitignore create mode 100644 distributed/certs.yml create mode 100644 distributed/client/compose.yml create mode 100644 distributed/server/compose.yml diff --git a/aiopslab/orchestrator/problems/flower/__init__.py b/aiopslab/orchestrator/problems/flower/__init__.py new file mode 100644 index 00000000..6eb3430a --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/__init__.py @@ -0,0 +1 @@ +from .flower_test import FlowerTest \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/flower_test.py b/aiopslab/orchestrator/problems/flower/flower_test.py new file mode 100644 index 00000000..4a55009b --- /dev/null +++ b/aiopslab/orchestrator/problems/flower/flower_test.py @@ -0,0 +1,9 @@ +from aiopslab.service.apps.flower import Flower + +class FlowerTest: + def __init__(self): + self.app = Flower() + + def start_workload(self): + print("== Start Workload ==") + self.app.deploy() \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/pyproject.toml b/aiopslab/orchestrator/problems/flower/pyproject.toml index a945fb91..b99b8fcd 100644 --- a/aiopslab/orchestrator/problems/flower/pyproject.toml +++ b/aiopslab/orchestrator/problems/flower/pyproject.toml @@ -36,5 +36,5 @@ default = "local-simulation" options.num-supernodes = 10 [tool.flwr.federations.local-deployment] -address = "127.0.0.1:9093" +address = "10.244.1.13:9093" insecure = true \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/registry.py b/aiopslab/orchestrator/problems/registry.py index 2563b27d..39be27f8 100644 --- a/aiopslab/orchestrator/problems/registry.py +++ b/aiopslab/orchestrator/problems/registry.py @@ -27,6 +27,7 @@ from aiopslab.orchestrator.problems.redeploy_without_pv import * from aiopslab.orchestrator.problems.wrong_bin_usage import * from aiopslab.orchestrator.problems.operator_misoperation import * +from aiopslab.orchestrator.problems.flower import * class ProblemRegistry: @@ -210,6 +211,8 @@ def __init__(self): "operator_security_context_fault-localization-1": K8SOperatorSecurityContextFaultLocalization, "operator_wrong_update_strategy-detection-1": K8SOperatorWrongUpdateStrategyDetection, "operator_wrong_update_strategy-localization-1": K8SOperatorWrongUpdateStrategyLocalization, + # Flower + "flower-test-1": FlowerTest, } def get_problem_instance(self, problem_id: str): diff --git a/aiopslab/service/apps/flower.py b/aiopslab/service/apps/flower.py index 3280c229..0ec7ba7a 100644 --- a/aiopslab/service/apps/flower.py +++ b/aiopslab/service/apps/flower.py @@ -65,5 +65,5 @@ def _remove_pv_finalizers(self, pv_name: str): if __name__ == "__main__": flower = Flower() flower.deploy() - flower.delete() - flower.cleanup() \ No newline at end of file + # flower.delete() + # flower.cleanup() \ No newline at end of file diff --git a/distributed/.gitignore b/distributed/.gitignore new file mode 100644 index 00000000..9c249f37 --- /dev/null +++ b/distributed/.gitignore @@ -0,0 +1,2 @@ +superlink-certificates +server/state diff --git a/distributed/certs.yml b/distributed/certs.yml new file mode 100644 index 00000000..0c8a3009 --- /dev/null +++ b/distributed/certs.yml @@ -0,0 +1,5 @@ +services: + gen-certs: + build: + args: + SUPERLINK_IP: ${SUPERLINK_IP:-127.0.0.1} diff --git a/distributed/client/compose.yml b/distributed/client/compose.yml new file mode 100644 index 00000000..9a20f02f --- /dev/null +++ b/distributed/client/compose.yml @@ -0,0 +1,130 @@ +services: + supernode-1: + image: flwr/supernode:${FLWR_VERSION:-1.16.0} + command: + - --superlink + - ${SUPERLINK_IP:-127.0.0.1}:9092 + - --clientappio-api-address + - 0.0.0.0:9094 + - --isolation + - process + - --node-config + - "partition-id=0 num-partitions=2" + - --root-certificates + - certificates/superlink-ca.crt + secrets: + - source: superlink-ca-certfile + target: /app/certificates/superlink-ca.crt + + supernode-2: + image: flwr/supernode:${FLWR_VERSION:-1.16.0} + command: + - --superlink + - ${SUPERLINK_IP:-127.0.0.1}:9092 + - --clientappio-api-address + - 0.0.0.0:9095 + - --isolation + - process + - --node-config + - "partition-id=1 num-partitions=2" + - --root-certificates + - certificates/superlink-ca.crt + secrets: + - source: superlink-ca-certfile + target: /app/certificates/superlink-ca.crt + + # uncomment to add another SuperNode + # + # supernode-3: + # image: flwr/supernode:${FLWR_VERSION:-1.16.0} + # command: + # - --superlink + # - ${SUPERLINK_IP:-127.0.0.1}:9092 + # - --clientappio-api-address + # - 0.0.0.0:9096 + # - --isolation + # - process + # - --node-config + # - "partition-id=1 num-partitions=2" + # - --root-certificates + # - certificates/superlink-ca.crt + # secrets: + # - source: superlink-ca-certfile + # target: /app/certificates/superlink-ca.crt + + clientapp-1: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-1:9094 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-1 + + clientapp-2: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-clientapp"] + command: + - --insecure + - --clientappio-api-address + - supernode-2:9095 + deploy: + resources: + limits: + cpus: "2" + stop_signal: SIGINT + depends_on: + - supernode-2 + # uncomment to add another ClientApp + # + # clientapp-3: + # build: + # context: ${PROJECT_DIR:-.} + # dockerfile_inline: | + # FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} + + # WORKDIR /app + # COPY --chown=app:app pyproject.toml . + # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + # && python -m pip install -U --no-cache-dir . + + # ENTRYPOINT ["flwr-clientapp"] + # command: + # - --insecure + # - --clientappio-api-address + # - supernode-3:9096 + # deploy: + # resources: + # limits: + # cpus: "2" + # stop_signal: SIGINT + # depends_on: + # - supernode-3 + +secrets: + superlink-ca-certfile: + file: ../superlink-certificates/ca.crt diff --git a/distributed/server/compose.yml b/distributed/server/compose.yml new file mode 100644 index 00000000..89f5de86 --- /dev/null +++ b/distributed/server/compose.yml @@ -0,0 +1,49 @@ +services: + superlink: + image: flwr/superlink:${FLWR_VERSION:-1.16.0} + command: + - --isolation + - process + - --ssl-ca-certfile=certificates/ca.crt + - --ssl-certfile=certificates/server.pem + - --ssl-keyfile=certificates/server.key + - --database=state/state.db + volumes: + - ./state/:/app/state/:rw + secrets: + - source: superlink-ca-certfile + target: /app/certificates/ca.crt + - source: superlink-certfile + target: /app/certificates/server.pem + - source: superlink-keyfile + target: /app/certificates/server.key + ports: + - 9092:9092 + - 9093:9093 + + serverapp: + build: + context: ${PROJECT_DIR:-.} + dockerfile_inline: | + FROM flwr/serverapp:${FLWR_VERSION:-1.16.0} + + WORKDIR /app + COPY --chown=app:app pyproject.toml . + RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ + && python -m pip install -U --no-cache-dir . + + ENTRYPOINT ["flwr-serverapp"] + command: + - --insecure + - --serverappio-api-address + - superlink:9091 + depends_on: + - superlink + +secrets: + superlink-ca-certfile: + file: ../superlink-certificates/ca.crt + superlink-certfile: + file: ../superlink-certificates/server.pem + superlink-keyfile: + file: ../superlink-certificates/server.key diff --git a/flower/kubernetes/serverapp/serverapp-pod.yaml b/flower/kubernetes/serverapp/serverapp-pod.yaml index 08edf1cf..667f3bb6 100644 --- a/flower/kubernetes/serverapp/serverapp-pod.yaml +++ b/flower/kubernetes/serverapp/serverapp-pod.yaml @@ -8,6 +8,8 @@ metadata: io.kompose.service: serverapp name: serverapp spec: + securityContext: + runAsUser: 0 containers: - args: - --insecure diff --git a/flower/kubernetes/superlink/superlink-deployment.yaml b/flower/kubernetes/superlink/superlink-deployment.yaml index 0f77f4ef..9dc15ce0 100644 --- a/flower/kubernetes/superlink/superlink-deployment.yaml +++ b/flower/kubernetes/superlink/superlink-deployment.yaml @@ -20,6 +20,8 @@ spec: labels: io.kompose.service: superlink spec: + securityContext: + runAsUser: 0 containers: - args: - --insecure From 5ec998100573a54ebfe891164d51144ebfc4be50 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Sun, 27 Apr 2025 23:07:54 +0530 Subject: [PATCH 08/18] added llama client with working fault --- aiopslab/generators/fault/inject_virtual.py | 13 ++ aiopslab/orchestrator/actions/base.py | 40 ++-- aiopslab/orchestrator/orchestrator.py | 39 ++-- .../orchestrator/problems/flower/__init__.py | 1 - .../problems/flower/flower_test.py | 9 - .../problems/flower/pyproject.toml | 40 ---- .../problems/flower/train/__init__.py | 1 - .../problems/flower/train/client_app.py | 55 ------ .../problems/flower/train/server_app.py | 31 --- .../problems/flower/train/task.py | 112 ----------- .../problems/flower_node_stop/__init__.py | 6 + .../problems/flower_node_stop/node_stop.py | 69 +++++++ aiopslab/orchestrator/problems/registry.py | 4 +- aiopslab/service/apps/base.py | 4 + aiopslab/service/apps/flower.py | 70 ++----- aiopslab/service/dock.py | 57 ++++++ aiopslab/service/metadata/flower.json | 4 +- aiopslab/service/shell.py | 15 +- clients/llama.py | 70 +++++++ clients/utils/llm.py | 44 +++++ flower/compose.yml | 186 ------------------ flower/config/pyproject.toml | 36 ---- .../clientapp/clientapp-1-deployment.yaml | 36 ---- .../clientapp/clientapp-1-service.yaml | 16 -- .../clientapp/clientapp-2-deployment.yaml | 36 ---- .../clientapp/clientapp-2-service.yaml | 16 -- .../kubernetes/serverapp/serverapp-pod.yaml | 23 --- .../serverapp/serverapp-service.yaml | 16 -- .../superlink/superlink-deployment.yaml | 35 ---- .../superlink/superlink-service.yaml | 16 -- .../supernode/supernode-1-deployment.yaml | 39 ---- .../supernode/supernode-1-service.yaml | 16 -- .../supernode/supernode-2-deployment.yaml | 39 ---- .../supernode/supernode-2-service.yaml | 16 -- 34 files changed, 336 insertions(+), 874 deletions(-) delete mode 100644 aiopslab/orchestrator/problems/flower/__init__.py delete mode 100644 aiopslab/orchestrator/problems/flower/flower_test.py delete mode 100644 aiopslab/orchestrator/problems/flower/pyproject.toml delete mode 100644 aiopslab/orchestrator/problems/flower/train/__init__.py delete mode 100644 aiopslab/orchestrator/problems/flower/train/client_app.py delete mode 100644 aiopslab/orchestrator/problems/flower/train/server_app.py delete mode 100644 aiopslab/orchestrator/problems/flower/train/task.py create mode 100644 aiopslab/orchestrator/problems/flower_node_stop/__init__.py create mode 100644 aiopslab/orchestrator/problems/flower_node_stop/node_stop.py create mode 100644 aiopslab/service/dock.py create mode 100644 clients/llama.py delete mode 100644 flower/compose.yml delete mode 100644 flower/config/pyproject.toml delete mode 100644 flower/kubernetes/clientapp/clientapp-1-deployment.yaml delete mode 100644 flower/kubernetes/clientapp/clientapp-1-service.yaml delete mode 100644 flower/kubernetes/clientapp/clientapp-2-deployment.yaml delete mode 100644 flower/kubernetes/clientapp/clientapp-2-service.yaml delete mode 100644 flower/kubernetes/serverapp/serverapp-pod.yaml delete mode 100644 flower/kubernetes/serverapp/serverapp-service.yaml delete mode 100644 flower/kubernetes/superlink/superlink-deployment.yaml delete mode 100644 flower/kubernetes/superlink/superlink-service.yaml delete mode 100644 flower/kubernetes/supernode/supernode-1-deployment.yaml delete mode 100644 flower/kubernetes/supernode/supernode-1-service.yaml delete mode 100644 flower/kubernetes/supernode/supernode-2-deployment.yaml delete mode 100644 flower/kubernetes/supernode/supernode-2-service.yaml diff --git a/aiopslab/generators/fault/inject_virtual.py b/aiopslab/generators/fault/inject_virtual.py index 8b6680d0..620a6c25 100644 --- a/aiopslab/generators/fault/inject_virtual.py +++ b/aiopslab/generators/fault/inject_virtual.py @@ -8,6 +8,7 @@ from aiopslab.service.kubectl import KubeCtl from aiopslab.service.helm import Helm +from aiopslab.service.dock import Docker from aiopslab.generators.fault.base import FaultInjector from aiopslab.service.apps.base import Application from aiopslab.paths import TARGET_MICROSERVICES @@ -18,6 +19,7 @@ def __init__(self, namespace: str): super().__init__(namespace) self.namespace = namespace self.kubectl = KubeCtl() + self.docker = Docker() self.mongo_service_pod_map = { "url-shorten-mongodb": "url-shorten-service", } @@ -248,6 +250,17 @@ def recover_wrong_bin_usage(self, microservices: list[str]): self.kubectl.exec_command(apply_command) print(f"Recovered from wrong binary usage fault for service: {service}") + + def inject_container_stop(self, microservices: list[str]): + """Inject a fault to stop a container.""" + for service in microservices: + self.docker.get_container(service).stop() + print(f"Stopped container {service}.") + + def recover_container_stop(self, microservices: list[str]): + for service in microservices: + self.docker.get_container(service).start() + print(f"Started container {service}.") ############# HELPER FUNCTIONS ################ def _wait_for_pods_ready(self, microservices: list[str], timeout: int = 30): diff --git a/aiopslab/orchestrator/actions/base.py b/aiopslab/orchestrator/actions/base.py index b623cca5..f3abb48c 100644 --- a/aiopslab/orchestrator/actions/base.py +++ b/aiopslab/orchestrator/actions/base.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta from aiopslab.utils.actions import action, read, write from aiopslab.service.kubectl import KubeCtl +from aiopslab.service.dock import Docker from aiopslab.service.shell import Shell # from aiopslab.observer import initialize_pod_and_service_lists @@ -22,7 +23,7 @@ class TaskActions: @read def get_logs(namespace: str, service: str) -> str: """ - Collects relevant log data from a pod using Kubectl. + Collects relevant log data from a pod using Kubectl or from a container with Docker. Args: namespace (str): The namespace in which the service is running. @@ -31,20 +32,29 @@ def get_logs(namespace: str, service: str) -> str: Returns: str | dict | list[dicts]: Log data as a structured object or a string. """ - kubectl = KubeCtl() - try: - if namespace == "test-social-network": - user_service_pod = kubectl.get_pod_name(namespace, f"app={service}") - elif namespace == "test-hotel-reservation": - user_service_pod = kubectl.get_pod_name( - namespace, f"io.kompose.service={service}" - ) - else: - raise Exception - logs = kubectl.get_pod_logs(user_service_pod, namespace) - except Exception as e: - return "Error: Your service/namespace does not exist. Use kubectl to check." - + if namespace == "docker": + docker = Docker() + try: + logs = docker.get_logs(service) + except Exception as e: + return "Error: Your service does not exist. Use docker to check." + + else: + kubectl = KubeCtl() + try: + if namespace == "test-social-network": + user_service_pod = kubectl.get_pod_name(namespace, f"app={service}") + elif namespace == "test-hotel-reservation": + user_service_pod = kubectl.get_pod_name( + namespace, f"io.kompose.service={service}" + ) + else: + raise Exception + logs = kubectl.get_pod_logs(user_service_pod, namespace) + except Exception as e: + return "Error: Your service/namespace does not exist. Use kubectl to check." + + print(logs) logs = "\n".join(logs.split("\n")) return logs diff --git a/aiopslab/orchestrator/orchestrator.py b/aiopslab/orchestrator/orchestrator.py index a1938f1d..705fe791 100644 --- a/aiopslab/orchestrator/orchestrator.py +++ b/aiopslab/orchestrator/orchestrator.py @@ -44,25 +44,26 @@ def init_problem(self, problem_id: str): self.session.set_problem(prob, pid=problem_id) self.session.set_agent(self.agent_name) - print("Setting up OpenEBS...") - - command = "kubectl get pods -n openebs" - result = self.kubectl.exec_command(command) - if "Running" in result: - print("OpenEBS is already running. Skipping installation.") - else: - self.kubectl.exec_command( - "kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml" - ) - self.kubectl.exec_command( - "kubectl patch storageclass openebs-hostpath -p '{\"metadata\": {\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'" - ) - self.kubectl.wait_for_ready("openebs") - print("OpenEBS setup completed.") - - # Setup and deploy Prometheus - self.prometheus = Prometheus() - self.prometheus.deploy() + if "flower" not in problem_id: # temporary fix for testing, will edit later + print("Setting up OpenEBS...") + + command = "kubectl get pods -n openebs" + result = self.kubectl.exec_command(command) + if "Running" in result: + print("OpenEBS is already running. Skipping installation.") + else: + self.kubectl.exec_command( + "kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml" + ) + self.kubectl.exec_command( + "kubectl patch storageclass openebs-hostpath -p '{\"metadata\": {\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'" + ) + self.kubectl.wait_for_ready("openebs") + print("OpenEBS setup completed.") + + # Setup and deploy Prometheus + self.prometheus = Prometheus() + self.prometheus.deploy() # deploy service prob.app.delete() diff --git a/aiopslab/orchestrator/problems/flower/__init__.py b/aiopslab/orchestrator/problems/flower/__init__.py deleted file mode 100644 index 6eb3430a..00000000 --- a/aiopslab/orchestrator/problems/flower/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .flower_test import FlowerTest \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/flower_test.py b/aiopslab/orchestrator/problems/flower/flower_test.py deleted file mode 100644 index 4a55009b..00000000 --- a/aiopslab/orchestrator/problems/flower/flower_test.py +++ /dev/null @@ -1,9 +0,0 @@ -from aiopslab.service.apps.flower import Flower - -class FlowerTest: - def __init__(self): - self.app = Flower() - - def start_workload(self): - print("== Start Workload ==") - self.app.deploy() \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/pyproject.toml b/aiopslab/orchestrator/problems/flower/pyproject.toml deleted file mode 100644 index b99b8fcd..00000000 --- a/aiopslab/orchestrator/problems/flower/pyproject.toml +++ /dev/null @@ -1,40 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "Flops" -version = "1.0.0" -description = "" -license = "Apache-2.0" -dependencies = [ - "flwr[simulation]>=1.15.2", - "flwr-datasets[vision]>=0.5.0", - "torch==2.5.1", - "torchvision==0.20.1", -] - -[tool.hatch.build.targets.wheel] -packages = ["."] - -[tool.flwr.app] -publisher = "flower" - -[tool.flwr.app.components] -serverapp = "train.server_app:app" -clientapp = "train.client_app:app" - -[tool.flwr.app.config] -num-server-rounds = 3 -fraction-fit = 0.5 -local-epochs = 1 - -[tool.flwr.federations] -default = "local-simulation" - -[tool.flwr.federations.local-simulation] -options.num-supernodes = 10 - -[tool.flwr.federations.local-deployment] -address = "10.244.1.13:9093" -insecure = true \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower/train/__init__.py b/aiopslab/orchestrator/problems/flower/train/__init__.py deleted file mode 100644 index 71160e37..00000000 --- a/aiopslab/orchestrator/problems/flower/train/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" diff --git a/aiopslab/orchestrator/problems/flower/train/client_app.py b/aiopslab/orchestrator/problems/flower/train/client_app.py deleted file mode 100644 index 10102508..00000000 --- a/aiopslab/orchestrator/problems/flower/train/client_app.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -import torch - -from flwr.client import ClientApp, NumPyClient -from flwr.common import Context -from train.task import Net, get_weights, load_data, set_weights, test, train - - -# Define Flower Client and client_fn -class FlowerClient(NumPyClient): - def __init__(self, net, trainloader, valloader, local_epochs): - self.net = net - self.trainloader = trainloader - self.valloader = valloader - self.local_epochs = local_epochs - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.net.to(self.device) - - def fit(self, parameters, config): - set_weights(self.net, parameters) - train_loss = train( - self.net, - self.trainloader, - self.local_epochs, - self.device, - ) - return ( - get_weights(self.net), - len(self.trainloader.dataset), - {"train_loss": train_loss}, - ) - - def evaluate(self, parameters, config): - set_weights(self.net, parameters) - loss, accuracy = test(self.net, self.valloader, self.device) - return loss, len(self.valloader.dataset), {"accuracy": accuracy} - - -def client_fn(context: Context): - # Load model and data - net = Net() - partition_id = context.node_config["partition-id"] - num_partitions = context.node_config["num-partitions"] - trainloader, valloader = load_data(partition_id, num_partitions) - local_epochs = context.run_config["local-epochs"] - - # Return Client instance - return FlowerClient(net, trainloader, valloader, local_epochs).to_client() - - -# Flower ClientApp -app = ClientApp( - client_fn, -) diff --git a/aiopslab/orchestrator/problems/flower/train/server_app.py b/aiopslab/orchestrator/problems/flower/train/server_app.py deleted file mode 100644 index 43c74bcb..00000000 --- a/aiopslab/orchestrator/problems/flower/train/server_app.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -from flwr.common import Context, ndarrays_to_parameters -from flwr.server import ServerApp, ServerAppComponents, ServerConfig -from flwr.server.strategy import FedAvg -from train.task import Net, get_weights - - -def server_fn(context: Context): - # Read from config - num_rounds = context.run_config["num-server-rounds"] - fraction_fit = context.run_config["fraction-fit"] - - # Initialize model parameters - ndarrays = get_weights(Net()) - parameters = ndarrays_to_parameters(ndarrays) - - # Define strategy - strategy = FedAvg( - fraction_fit=fraction_fit, - fraction_evaluate=1.0, - min_available_clients=2, - initial_parameters=parameters, - ) - config = ServerConfig(num_rounds=num_rounds) - - return ServerAppComponents(strategy=strategy, config=config) - - -# Create ServerApp -app = ServerApp(server_fn=server_fn) diff --git a/aiopslab/orchestrator/problems/flower/train/task.py b/aiopslab/orchestrator/problems/flower/train/task.py deleted file mode 100644 index 9cabd9b8..00000000 --- a/aiopslab/orchestrator/problems/flower/train/task.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Flops: A Flower / PyTorch app.""" - -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F -from flwr_datasets import FederatedDataset -from flwr_datasets.partitioner import IidPartitioner -from torch.utils.data import DataLoader -from torchvision.transforms import Compose, Normalize, ToTensor - - -class Net(nn.Module): - """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" - - def __init__(self): - super(Net, self).__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(-1, 16 * 5 * 5) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - return self.fc3(x) - - -fds = None # Cache FederatedDataset - - -def load_data(partition_id: int, num_partitions: int): - """Load partition CIFAR10 data.""" - # Only initialize `FederatedDataset` once - global fds - if fds is None: - partitioner = IidPartitioner(num_partitions=num_partitions) - fds = FederatedDataset( - dataset="uoft-cs/cifar10", - partitioners={"train": partitioner}, - ) - partition = fds.load_partition(partition_id) - # Divide data on each node: 80% train, 20% test - partition_train_test = partition.train_test_split(test_size=0.2, seed=42) - pytorch_transforms = Compose( - [ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - - def apply_transforms(batch): - """Apply transforms to the partition from FederatedDataset.""" - batch["img"] = [pytorch_transforms(img) for img in batch["img"]] - return batch - - partition_train_test = partition_train_test.with_transform(apply_transforms) - trainloader = DataLoader(partition_train_test["train"], batch_size=32, shuffle=True) - testloader = DataLoader(partition_train_test["test"], batch_size=32) - return trainloader, testloader - - -def train(net, trainloader, epochs, device): - """Train the model on the training set.""" - net.to(device) # move model to GPU if available - criterion = torch.nn.CrossEntropyLoss().to(device) - optimizer = torch.optim.Adam(net.parameters(), lr=0.01) - net.train() - running_loss = 0.0 - for _ in range(epochs): - for batch in trainloader: - images = batch["img"] - labels = batch["label"] - optimizer.zero_grad() - loss = criterion(net(images.to(device)), labels.to(device)) - loss.backward() - optimizer.step() - running_loss += loss.item() - - avg_trainloss = running_loss / len(trainloader) - return avg_trainloss - - -def test(net, testloader, device): - """Validate the model on the test set.""" - net.to(device) - criterion = torch.nn.CrossEntropyLoss() - correct, loss = 0, 0.0 - with torch.no_grad(): - for batch in testloader: - images = batch["img"].to(device) - labels = batch["label"].to(device) - outputs = net(images) - loss += criterion(outputs, labels).item() - correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() - accuracy = correct / len(testloader.dataset) - loss = loss / len(testloader) - return loss, accuracy - - -def get_weights(net): - print('testing') - return [val.cpu().numpy() for _, val in net.state_dict().items()] - - -def set_weights(net, parameters): - params_dict = zip(net.state_dict().keys(), parameters) - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict}) - net.load_state_dict(state_dict, strict=True) diff --git a/aiopslab/orchestrator/problems/flower_node_stop/__init__.py b/aiopslab/orchestrator/problems/flower_node_stop/__init__.py new file mode 100644 index 00000000..fb33063c --- /dev/null +++ b/aiopslab/orchestrator/problems/flower_node_stop/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from .node_stop import ( + FlowerNodeStopDetection +) \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower_node_stop/node_stop.py b/aiopslab/orchestrator/problems/flower_node_stop/node_stop.py new file mode 100644 index 00000000..79596976 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower_node_stop/node_stop.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Docker node stop fault problem in the Flower application.""" + +from typing import Any + +from aiopslab.orchestrator.tasks import * +from aiopslab.service.dock import Docker +from aiopslab.service.apps.flower import Flower +from aiopslab.paths import TARGET_MICROSERVICES +from aiopslab.session import SessionItem +from aiopslab.generators.fault.inject_virtual import VirtualizationFaultInjector + + +class FlowerNodeStopBaseTask: + def __init__(self, faulty_service: str = "user-service"): + self.app = Flower() + self.docker = Docker() + self.namespace = self.app.namespace + self.faulty_service = faulty_service + self.train_dir = TARGET_MICROSERVICES / "flower" + + def start_workload(self): + print("== Start Workload ==") + command = "flwr run train local-deployment" + self.docker.exec_command(command, cwd=self.train_dir) + + def inject_fault(self): + print("== Fault Injection ==") + injector = VirtualizationFaultInjector(namespace=self.namespace) + injector._inject( + fault_type="container_stop", + microservices=[self.faulty_service], + ) + print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + + def recover_fault(self): + print("== Fault Recovery ==") + injector = VirtualizationFaultInjector(namespace=self.namespace) + injector._recover( + fault_type="container_stop", + microservices=[self.faulty_service], + ) + print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + + +################## Detection Problem ################## +class FlowerNodeStopDetection(FlowerNodeStopBaseTask, DetectionTask): + def __init__(self, faulty_service: str = "supernode-1"): + FlowerNodeStopBaseTask.__init__(self, faulty_service=faulty_service) + DetectionTask.__init__(self, self.app) + + def eval(self, soln: Any, trace: list[SessionItem], duration: float): + print("== Evaluation ==") + expected_solution = "Yes" + + if isinstance(soln, str): + if soln.strip().lower() == expected_solution.lower(): + print(f"Correct detection: {soln}") + self.add_result("Detection Accuracy", "Correct") + else: + print(f"Incorrect detection: {soln}") + self.add_result("Detection Accuracy", "Incorrect") + else: + print("Invalid solution format") + self.add_result("Detection Accuracy", "Invalid Format") + + return super().eval(soln, trace, duration) diff --git a/aiopslab/orchestrator/problems/registry.py b/aiopslab/orchestrator/problems/registry.py index 39be27f8..f91bd418 100644 --- a/aiopslab/orchestrator/problems/registry.py +++ b/aiopslab/orchestrator/problems/registry.py @@ -27,7 +27,7 @@ from aiopslab.orchestrator.problems.redeploy_without_pv import * from aiopslab.orchestrator.problems.wrong_bin_usage import * from aiopslab.orchestrator.problems.operator_misoperation import * -from aiopslab.orchestrator.problems.flower import * +from aiopslab.orchestrator.problems.flower_node_stop import * class ProblemRegistry: @@ -212,7 +212,7 @@ def __init__(self): "operator_wrong_update_strategy-detection-1": K8SOperatorWrongUpdateStrategyDetection, "operator_wrong_update_strategy-localization-1": K8SOperatorWrongUpdateStrategyLocalization, # Flower - "flower-test-1": FlowerTest, + "flower_node_stop-detection": FlowerNodeStopDetection, } def get_problem_instance(self, problem_id: str): diff --git a/aiopslab/service/apps/base.py b/aiopslab/service/apps/base.py index 214a8e38..f5cb6231 100644 --- a/aiopslab/service/apps/base.py +++ b/aiopslab/service/apps/base.py @@ -15,6 +15,7 @@ def __init__(self, config_file: str): self.helm_deploy = True self.helm_configs = {} self.k8s_deploy_path = None + self.docker_deploy_path = None def load_app_json(self): """Load (basic) application metadata into attributes. @@ -34,6 +35,9 @@ def load_app_json(self): if "K8S Deploy Path" in metadata: self.k8s_deploy_path = TARGET_MICROSERVICES / metadata["K8S Deploy Path"] + + if "Docker Deploy Path" in metadata: + self.docker_deploy_path = TARGET_MICROSERVICES / metadata["Docker Deploy Path"] def get_app_json(self) -> dict: """Get application metadata in JSON format. diff --git a/aiopslab/service/apps/flower.py b/aiopslab/service/apps/flower.py index 0ec7ba7a..22834d3e 100644 --- a/aiopslab/service/apps/flower.py +++ b/aiopslab/service/apps/flower.py @@ -1,69 +1,29 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../'))) - -import time -from aiopslab.service.kubectl import KubeCtl +from aiopslab.service.dock import Docker from aiopslab.service.apps.base import Application -from aiopslab.paths import FAULT_SCRIPTS, FLOWER_METADATA +from aiopslab.paths import FLOWER_METADATA class Flower(Application): def __init__(self): super().__init__(FLOWER_METADATA) - self.kubectl = KubeCtl() - self.script_dir = FAULT_SCRIPTS - self.helm_deploy = False - + self.docker = Docker() + self.load_app_json() - self.create_namespace() - - def load_app_json(self): - super().load_app_json() - metadata = self.get_app_json() - self.frontend_service = None - self.frontend_port = None def deploy(self): - """Deploy the Kubernetes configurations.""" - print(f"Deploying Kubernetes configurations in namespace: {self.namespace}") - self.kubectl.apply_configs(self.namespace, self.k8s_deploy_path) - self.kubectl.wait_for_ready(self.namespace) - + """Deploy the docker compose file.""" + print("Deploying docker compose files") + self.docker.compose_up(self.docker_deploy_path) + def delete(self): - """Delete the configmap.""" - self.kubectl.delete_configs(self.namespace, self.k8s_deploy_path) - + """Stop the docker containers.""" + print("Stopping the docker containers") + self.docker.compose_down(self.docker_deploy_path) + def cleanup(self): - """Delete the entire namespace for the flower application.""" - self.kubectl.delete_namespace(self.namespace) - time.sleep(10) - pvs = self.kubectl.exec_command( - "kubectl get pv --no-headers | grep 'test-flower' | awk '{print $1}'" - ).splitlines() - - for pv in pvs: - # Check if the PV is in a 'Terminating' state and remove the finalizers if necessary - self._remove_pv_finalizers(pv) - delete_command = f"kubectl delete pv {pv}" - delete_result = self.kubectl.exec_command(delete_command) - print(f"Deleted PersistentVolume {pv}: {delete_result.strip()}") - time.sleep(5) - - def _remove_pv_finalizers(self, pv_name: str): - """Remove finalizers from the PersistentVolume to prevent it from being stuck in a 'Terminating' state.""" - # Patch the PersistentVolume to remove finalizers if it is stuck - patch_command = ( - f'kubectl patch pv {pv_name} -p \'{{"metadata":{{"finalizers":null}}}}\'' - ) - _ = self.kubectl.exec_command(patch_command) - - -if __name__ == "__main__": - flower = Flower() - flower.deploy() - # flower.delete() - # flower.cleanup() \ No newline at end of file + """Delete all stopped docker containers.""" + print("Deleting stopped containers") + self.docker.cleanup() diff --git a/aiopslab/service/dock.py b/aiopslab/service/dock.py new file mode 100644 index 00000000..149ff1cc --- /dev/null +++ b/aiopslab/service/dock.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Interface to Docker controller service.""" + +import docker +import subprocess + + +class Docker: + def __init__(self): + self.client = docker.from_env() + + def list_containers(self): + """Get all containers.""" + return self.client.containers.list() + + def get_container(self, container_id): + """Get a container by ID.""" + return self.client.containers.get(container_id) + + def get_logs(self, container_id): + """Get logs for a container.""" + return self.get_container(container_id).logs().decode("utf-8") + + def compose_up(self, cwd): + """Run docker-compose up.""" + command = "docker compose up -d" + return self.exec_command(command, cwd=cwd) + + def compose_down(self, cwd): + """Run docker-compose down.""" + command = "docker compose down" + return self.exec_command(command, cwd=cwd) + + def cleanup(self): + """Remove the stopped docker containers.""" + command = "docker container prune -f" + return self.exec_command(command) + + def exec_command(self, command: str, input_data=None, cwd=None): + """Execute an arbitrary command.""" + if input_data is not None: + input_data = input_data.encode("utf-8") + try: + out = subprocess.run( + command, + input=input_data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + cwd=cwd, + ) + if out is not None: + return out.stdout.decode("utf-8") + except subprocess.CalledProcessError as e: + return e.stderr.decode("utf-8") diff --git a/aiopslab/service/metadata/flower.json b/aiopslab/service/metadata/flower.json index 15b77674..e6def88e 100644 --- a/aiopslab/service/metadata/flower.json +++ b/aiopslab/service/metadata/flower.json @@ -1,10 +1,10 @@ { "Name": "Flower", - "Namespace": "test-flower", + "Namespace": "docker", "Desc": "A federated learning application to train models on edge devices and aggregate them on a central server.", "Supported Operations": [ "Train models on edge devices", "Aggregate models on a central server" ], - "K8S Deploy Path": "flower/kubernetes" + "Docker Deploy Path": "flower" } \ No newline at end of file diff --git a/aiopslab/service/shell.py b/aiopslab/service/shell.py index 2b894caa..6721cf53 100644 --- a/aiopslab/service/shell.py +++ b/aiopslab/service/shell.py @@ -20,17 +20,19 @@ class Shell: def exec(command: str, input_data=None, cwd=None): """Execute a shell command on localhost, via SSH, or inside kind's control-plane container.""" k8s_host = config.get("k8s_host", "localhost") # Default to localhost + + print("Command:", command) if k8s_host == "kind": print("[INFO] Running command inside kind-control-plane Docker container.") return Shell.docker_exec("kind-control-plane", command) elif k8s_host == "localhost": - print( - "[WARNING] Running commands on localhost is not recommended. " - "This may pose safety and security risks when using an AI agent locally. " - "I hope you know what you're doing!!!" - ) + # print( + # "[WARNING] Running commands on localhost is not recommended. " + # "This may pose safety and security risks when using an AI agent locally. " + # "I hope you know what you're doing!!!" + # ) return Shell.local_exec(command, input_data, cwd) else: @@ -51,9 +53,10 @@ def local_exec(command: str, input_data=None, cwd=None): stderr=subprocess.PIPE, shell=True, cwd=cwd, + timeout=10, # need to account for this properly ) - if out.stderr or out.returncode != 0: + if out.returncode != 0: error_message = out.stderr.decode("utf-8") print(f"[ERROR] Command execution failed: {error_message}") return error_message diff --git a/clients/llama.py b/clients/llama.py new file mode 100644 index 00000000..80f8887f --- /dev/null +++ b/clients/llama.py @@ -0,0 +1,70 @@ +""" +Naive LLaMA client (with shell access) for AIOpsLab. +""" + +import sys +sys.path.append("..") +sys.path.append("../..") + +import asyncio + +from aiopslab.orchestrator import Orchestrator +from clients.utils.llm import LLaMA3 +from clients.utils.templates import DOCS + + +class Agent: + def __init__(self): + self.history = [] + self.llm = LLaMA3() + + def init_context(self, problem_desc: str, instructions: str, apis: str): + """Initialize the context for the agent.""" + + self.telemetry_apis = self._filter_dict(apis, lambda k, _: "get_logs" in k) + self.shell_api = self._filter_dict(apis, lambda k, _: "exec_shell" in k) + self.submit_api = self._filter_dict(apis, lambda k, _: "submit" in k) + stringify_apis = lambda apis: "\n\n".join( + [f"{k}\n{v}" for k, v in apis.items()] + ) + + self.system_message = DOCS.format( + prob_desc=problem_desc, + telemetry_apis=stringify_apis(self.telemetry_apis), + shell_api=stringify_apis(self.shell_api), + submit_api=stringify_apis(self.submit_api), + ) + + self.task_message = instructions + + self.history.append({"role": "system", "content": self.system_message}) + self.history.append({"role": "user", "content": self.task_message}) + + async def get_action(self, input) -> str: + """Wrapper to interface the agent with OpsBench. + + Args: + input (str): The input from the orchestrator/environment. + + Returns: + str: The response from the agent. + """ + self.history.append({"role": "user", "content": input}) + response = self.llm.run(self.history) + self.history.append({"role": "assistant", "content": response[0]}) + return response[0] + + def _filter_dict(self, dictionary, filter_func): + return {k: v for k, v in dictionary.items() if filter_func(k, v)} + + +if __name__ == "__main__": + agent = Agent() + + orchestrator = Orchestrator() + orchestrator.register_agent(agent, name="llama-w-shell") + + pid = "flower_node_stop-detection" + problem_desc, instructs, apis = orchestrator.init_problem(pid) + agent.init_context(problem_desc, instructs, apis) + asyncio.run(orchestrator.start_problem(max_steps=10)) diff --git a/clients/utils/llm.py b/clients/utils/llm.py index 1a156a36..833c7059 100644 --- a/clients/utils/llm.py +++ b/clients/utils/llm.py @@ -5,6 +5,7 @@ import os from openai import OpenAI +from groq import Groq from pathlib import Path import json @@ -82,3 +83,46 @@ def run(self, payload: list[dict[str, str]]) -> list[str]: self.cache.add_to_cache(payload, response) self.cache.save_cache() return response + + +class LLaMA3: + """Abstraction for Meta's LLaMA-3 model.""" + + def __init__(self): + self.cache = Cache() + + def inference(self, payload: list[dict[str, str]]) -> list[str]: + if self.cache is not None: + cache_result = self.cache.get_from_cache(payload) + if cache_result is not None: + return cache_result + + client = Groq(api_key=os.getenv("GROQ_API_KEY")) + try: + response = client.chat.completions.create( + messages=payload, + model="llama-3.1-8b-instant", + max_tokens=1024, + temperature=0.5, + top_p=0.95, + frequency_penalty=0.0, + presence_penalty=0.0, + n=1, + timeout=60, + stop=[], + ) + except Exception as e: + print(f"Exception: {repr(e)}") + raise e + + return [c.message.content for c in response.choices] # type: ignore + + def run(self, payload: list[dict[str, str]]) -> list[str]: + response = self.inference(payload) + + print("Llama response:", response) + + if self.cache is not None: + self.cache.add_to_cache(payload, response) + self.cache.save_cache() + return response diff --git a/flower/compose.yml b/flower/compose.yml deleted file mode 100644 index 8753ddbc..00000000 --- a/flower/compose.yml +++ /dev/null @@ -1,186 +0,0 @@ -services: - # create a SuperLink service - superlink: - image: flwr/superlink:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --isolation - - process - ports: - - 9093:9093 - - # create a ServerApp service - serverapp: - build: - context: config - dockerfile_inline: | - FROM flwr/serverapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-serverapp"] - command: - - --insecure - - --serverappio-api-address - - superlink:9091 - restart: on-failure - depends_on: - - superlink - - # create two SuperNode services with different node configs - supernode-1: - image: flwr/supernode:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9094 - - --isolation - - process - - --node-config - - "partition-id=0 num-partitions=2" - depends_on: - - superlink - - supernode-2: - image: flwr/supernode:${FLWR_VERSION:-1.15.2} - command: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9095 - - --isolation - - process - - --node-config - - "partition-id=1 num-partitions=2" - depends_on: - - superlink - - # uncomment to add another SuperNode - # - # supernode-3: - # image: flwr/supernode:${FLWR_VERSION:-1.15.2} - # command: - # - --insecure - # - --superlink - # - superlink:9092 - # - --clientappio-api-address - # - 0.0.0.0:9096 - # - --isolation - # - process - # - --node-config - # - "partition-id=1 num-partitions=2" - # depends_on: - # - superlink - - # create two ClientApp services - clientapp-1: - build: - context: config - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-1:9094 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-1 - - clientapp-2: - build: - context: config - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # gcc is required for the fastai quickstart example - USER root - RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - USER app - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-2:9095 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-2 - - # uncomment to add another ClientApp - # - # clientapp-3: - # build: - # context: config - # dockerfile_inline: | - # FROM flwr/clientapp:${FLWR_VERSION:-1.15.2} - - # # gcc is required for the fastai quickstart example - # USER root - # RUN apt-get update \ - # && apt-get -y --no-install-recommends install \ - # build-essential \ - # && rm -rf /var/lib/apt/lists/* - # USER app - - # WORKDIR /app - # COPY --chown=app:app pyproject.toml . - # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - # && python -m pip install -U --no-cache-dir . - - # ENTRYPOINT ["flwr-clientapp"] - # command: - # - --insecure - # - --clientappio-api-address - # - supernode-3:9096 - # deploy: - # resources: - # limits: - # cpus: "2" - # stop_signal: SIGINT - # depends_on: - # - supernode-3 diff --git a/flower/config/pyproject.toml b/flower/config/pyproject.toml deleted file mode 100644 index b17fa6da..00000000 --- a/flower/config/pyproject.toml +++ /dev/null @@ -1,36 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "Flops" -version = "1.0.0" -description = "" -license = "Apache-2.0" -dependencies = [ - "flwr[simulation]>=1.15.2", - "flwr-datasets[vision]>=0.5.0", - "torch==2.5.1", - "torchvision==0.20.1", -] - -[tool.hatch.build.targets.wheel] -packages = ["."] - -[tool.flwr.app] -publisher = "flower" - -[tool.flwr.app.components] -serverapp = "train.server_app:app" -clientapp = "train.client_app:app" - -[tool.flwr.app.config] -num-server-rounds = 3 -fraction-fit = 0.5 -local-epochs = 1 - -[tool.flwr.federations] -default = "local-simulation" - -[tool.flwr.federations.local-simulation] -options.num-supernodes = 10 \ No newline at end of file diff --git a/flower/kubernetes/clientapp/clientapp-1-deployment.yaml b/flower/kubernetes/clientapp/clientapp-1-deployment.yaml deleted file mode 100644 index 4759f0a4..00000000 --- a/flower/kubernetes/clientapp/clientapp-1-deployment.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-1 - name: clientapp-1 -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: clientapp-1 - template: - metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-1 - spec: - containers: - - args: - - --insecure - - --clientappio-api-address - - supernode-1:9094 - image: adityapgupta/flower-client-1:latest - name: clientapp-1 - ports: - - containerPort: 9094 - protocol: TCP - resources: - limits: - cpu: "2" - restartPolicy: Always diff --git a/flower/kubernetes/clientapp/clientapp-1-service.yaml b/flower/kubernetes/clientapp/clientapp-1-service.yaml deleted file mode 100644 index eb26aeaf..00000000 --- a/flower/kubernetes/clientapp/clientapp-1-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-1 - name: clientapp-1 -spec: - ports: - - name: "9094" - port: 9094 - targetPort: 9094 - selector: - io.kompose.service: clientapp-1 diff --git a/flower/kubernetes/clientapp/clientapp-2-deployment.yaml b/flower/kubernetes/clientapp/clientapp-2-deployment.yaml deleted file mode 100644 index 09abaabc..00000000 --- a/flower/kubernetes/clientapp/clientapp-2-deployment.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-2 - name: clientapp-2 -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: clientapp-2 - template: - metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-2 - spec: - containers: - - args: - - --insecure - - --clientappio-api-address - - supernode-2:9095 - image: adityapgupta/flower-client-2:latest - name: clientapp-2 - ports: - - containerPort: 9095 - protocol: TCP - resources: - limits: - cpu: "2" - restartPolicy: Always diff --git a/flower/kubernetes/clientapp/clientapp-2-service.yaml b/flower/kubernetes/clientapp/clientapp-2-service.yaml deleted file mode 100644 index 07c40063..00000000 --- a/flower/kubernetes/clientapp/clientapp-2-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: clientapp-2 - name: clientapp-2 -spec: - ports: - - name: "9095" - port: 9095 - targetPort: 9095 - selector: - io.kompose.service: clientapp-2 diff --git a/flower/kubernetes/serverapp/serverapp-pod.yaml b/flower/kubernetes/serverapp/serverapp-pod.yaml deleted file mode 100644 index 667f3bb6..00000000 --- a/flower/kubernetes/serverapp/serverapp-pod.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: serverapp - name: serverapp -spec: - securityContext: - runAsUser: 0 - containers: - - args: - - --insecure - - --serverappio-api-address - - superlink:9091 - image: adityapgupta/flower-server:latest - name: serverapp - ports: - - containerPort: 9091 - protocol: TCP - restartPolicy: OnFailure diff --git a/flower/kubernetes/serverapp/serverapp-service.yaml b/flower/kubernetes/serverapp/serverapp-service.yaml deleted file mode 100644 index bf618e47..00000000 --- a/flower/kubernetes/serverapp/serverapp-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: serverapp - name: serverapp -spec: - ports: - - name: "9091" - port: 9091 - targetPort: 9091 - selector: - io.kompose.service: serverapp diff --git a/flower/kubernetes/superlink/superlink-deployment.yaml b/flower/kubernetes/superlink/superlink-deployment.yaml deleted file mode 100644 index 9dc15ce0..00000000 --- a/flower/kubernetes/superlink/superlink-deployment.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: superlink - name: superlink -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: superlink - template: - metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: superlink - spec: - securityContext: - runAsUser: 0 - containers: - - args: - - --insecure - - --isolation - - process - image: flwr/superlink:1.15.2 - name: superlink - ports: - - containerPort: 9093 - protocol: TCP - restartPolicy: Always diff --git a/flower/kubernetes/superlink/superlink-service.yaml b/flower/kubernetes/superlink/superlink-service.yaml deleted file mode 100644 index 114c3ea5..00000000 --- a/flower/kubernetes/superlink/superlink-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: superlink - name: superlink -spec: - ports: - - name: "9093" - port: 9093 - targetPort: 9093 - selector: - io.kompose.service: superlink diff --git a/flower/kubernetes/supernode/supernode-1-deployment.yaml b/flower/kubernetes/supernode/supernode-1-deployment.yaml deleted file mode 100644 index c1957f9e..00000000 --- a/flower/kubernetes/supernode/supernode-1-deployment.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-1 - name: supernode-1 -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: supernode-1 - template: - metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-1 - spec: - containers: - - args: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9094 - - --isolation - - process - - --node-config - - partition-id=0 num-partitions=2 - image: flwr/supernode:1.15.2 - name: supernode-1 - ports: - - containerPort: 9094 - protocol: TCP - restartPolicy: Always diff --git a/flower/kubernetes/supernode/supernode-1-service.yaml b/flower/kubernetes/supernode/supernode-1-service.yaml deleted file mode 100644 index aa744e3f..00000000 --- a/flower/kubernetes/supernode/supernode-1-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-1 - name: supernode-1 -spec: - ports: - - name: "9094" - port: 9094 - targetPort: 9094 - selector: - io.kompose.service: supernode-1 diff --git a/flower/kubernetes/supernode/supernode-2-deployment.yaml b/flower/kubernetes/supernode/supernode-2-deployment.yaml deleted file mode 100644 index 5ea2a4d3..00000000 --- a/flower/kubernetes/supernode/supernode-2-deployment.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-2 - name: supernode-2 -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: supernode-2 - template: - metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-2 - spec: - containers: - - args: - - --insecure - - --superlink - - superlink:9092 - - --clientappio-api-address - - 0.0.0.0:9095 - - --isolation - - process - - --node-config - - partition-id=1 num-partitions=2 - image: flwr/supernode:1.15.2 - name: supernode-2 - ports: - - containerPort: 9095 - protocol: TCP - restartPolicy: Always diff --git a/flower/kubernetes/supernode/supernode-2-service.yaml b/flower/kubernetes/supernode/supernode-2-service.yaml deleted file mode 100644 index 2de7487c..00000000 --- a/flower/kubernetes/supernode/supernode-2-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert - kompose.version: 1.35.0 (9532ceef3) - labels: - io.kompose.service: supernode-2 - name: supernode-2 -spec: - ports: - - name: "9095" - port: 9095 - targetPort: 9095 - selector: - io.kompose.service: supernode-2 From e27416b632c5901193a912ac81b2bfe432d7349b Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Thu, 1 May 2025 23:03:34 +0530 Subject: [PATCH 09/18] added delay to supernode stop --- aiopslab/generators/fault/inject_virtual.py | 4 ++++ aiopslab/orchestrator/actions/base.py | 3 +++ aiopslab/orchestrator/orchestrator.py | 16 +++++++++------- aiopslab/service/shell.py | 2 -- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/aiopslab/generators/fault/inject_virtual.py b/aiopslab/generators/fault/inject_virtual.py index 620a6c25..af9fafc6 100644 --- a/aiopslab/generators/fault/inject_virtual.py +++ b/aiopslab/generators/fault/inject_virtual.py @@ -256,6 +256,10 @@ def inject_container_stop(self, microservices: list[str]): for service in microservices: self.docker.get_container(service).stop() print(f"Stopped container {service}.") + + print("Waiting for faults to propagate...") + time.sleep(15) + print("Faults propagated.") def recover_container_stop(self, microservices: list[str]): for service in microservices: diff --git a/aiopslab/orchestrator/actions/base.py b/aiopslab/orchestrator/actions/base.py index 326cebd4..2aa9d49f 100644 --- a/aiopslab/orchestrator/actions/base.py +++ b/aiopslab/orchestrator/actions/base.py @@ -81,6 +81,9 @@ def exec_shell(command: str) -> str: """ if "kubectl edit" in command or "edit svc" in command: return "Error: Cannot use `kubectl edit`. Use `kubectl patch` instead." + + if "docker logs -f" in command: + return "Error: Cannot use `docker logs -f`. Use `docker logs` instead." return Shell.exec(command) diff --git a/aiopslab/orchestrator/orchestrator.py b/aiopslab/orchestrator/orchestrator.py index 67da630b..c29b68b1 100644 --- a/aiopslab/orchestrator/orchestrator.py +++ b/aiopslab/orchestrator/orchestrator.py @@ -201,13 +201,15 @@ async def start_problem(self, max_steps: int): # But this will take more time. # if not self.session.problem.sys_status_after_recovery(): self.session.problem.app.cleanup() - self.prometheus.teardown() - print("Uninstalling OpenEBS...") - self.kubectl.exec_command("kubectl delete sc openebs-hostpath openebs-device --ignore-not-found") - self.kubectl.exec_command( - "kubectl delete -f https://openebs.github.io/charts/openebs-operator.yaml" - ) - self.kubectl.wait_for_namespace_deletion("openebs") + + if self.session.problem.namespace != "docker": + self.prometheus.teardown() + print("Uninstalling OpenEBS...") + self.kubectl.exec_command("kubectl delete sc openebs-hostpath openebs-device --ignore-not-found") + self.kubectl.exec_command( + "kubectl delete -f https://openebs.github.io/charts/openebs-operator.yaml" + ) + self.kubectl.wait_for_namespace_deletion("openebs") self.execution_end_time = time.time() total_execution_time = self.execution_end_time - self.execution_start_time diff --git a/aiopslab/service/shell.py b/aiopslab/service/shell.py index 0f97f920..83d8fec8 100644 --- a/aiopslab/service/shell.py +++ b/aiopslab/service/shell.py @@ -21,8 +21,6 @@ def exec(command: str, input_data=None, cwd=None): """Execute a shell command on localhost, via SSH, or inside kind's control-plane container.""" k8s_host = config.get("k8s_host", "localhost") # Default to localhost - print("Command:", command) - if k8s_host == "kind": return Shell.docker_exec("kind-control-plane", command) From bf71fcd460bf02434dccbb3024e037feaaa9abd9 Mon Sep 17 00:00:00 2001 From: Armxyz1 Date: Thu, 1 May 2025 23:14:18 +0530 Subject: [PATCH 10/18] Adding LLaMa client --- clients/llama.py | 4 ++-- clients/utils/llm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/llama.py b/clients/llama.py index 80f8887f..c0e9c13b 100644 --- a/clients/llama.py +++ b/clients/llama.py @@ -9,14 +9,14 @@ import asyncio from aiopslab.orchestrator import Orchestrator -from clients.utils.llm import LLaMA3 +from clients.utils.llm import LLaMAClient from clients.utils.templates import DOCS class Agent: def __init__(self): self.history = [] - self.llm = LLaMA3() + self.llm = LLaMAClient() def init_context(self, problem_desc: str, instructions: str, apis: str): """Initialize the context for the agent.""" diff --git a/clients/utils/llm.py b/clients/utils/llm.py index 5fb1febf..e678d538 100644 --- a/clients/utils/llm.py +++ b/clients/utils/llm.py @@ -222,7 +222,7 @@ def run(self, payload: list[dict[str, str]]) -> list[str]: return response -class LLaMA3: +class LLaMAClient: """Abstraction for Meta's LLaMA-3 model.""" def __init__(self): From ee0beda6f57a034d6a9d4ab8e96611befc319a43 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Fri, 2 May 2025 14:03:02 +0530 Subject: [PATCH 11/18] edited flower description --- aiopslab/service/metadata/flower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiopslab/service/metadata/flower.json b/aiopslab/service/metadata/flower.json index e6def88e..87653228 100644 --- a/aiopslab/service/metadata/flower.json +++ b/aiopslab/service/metadata/flower.json @@ -1,7 +1,7 @@ { "Name": "Flower", "Namespace": "docker", - "Desc": "A federated learning application to train models on edge devices and aggregate them on a central server.", + "Desc": "A federated learning application to train models on edge devices and aggregate them on a central server. It consists of four main components, the serverapp which is the central server, the superlink which is the communication link for the serverapp, the clientapp which is the edge device application, and the supernode which is the communication link for the clientapp. The serverapp and superlink are used to aggregate models from the clientapp and supernode, while the clientapp and supernode are used to train models on edge devices. The current deployment consists of one server and two clients.", "Supported Operations": [ "Train models on edge devices", "Aggregate models on a central server" From 00bc2da7687a41dc9df699f902c2aecca8d2ac78 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Thu, 8 May 2025 15:27:34 +0530 Subject: [PATCH 12/18] cleanup --- distributed/.gitignore | 2 - distributed/certs.yml | 5 -- distributed/client/compose.yml | 130 --------------------------------- distributed/server/compose.yml | 49 ------------- 4 files changed, 186 deletions(-) delete mode 100644 distributed/.gitignore delete mode 100644 distributed/certs.yml delete mode 100644 distributed/client/compose.yml delete mode 100644 distributed/server/compose.yml diff --git a/distributed/.gitignore b/distributed/.gitignore deleted file mode 100644 index 9c249f37..00000000 --- a/distributed/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -superlink-certificates -server/state diff --git a/distributed/certs.yml b/distributed/certs.yml deleted file mode 100644 index 0c8a3009..00000000 --- a/distributed/certs.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - gen-certs: - build: - args: - SUPERLINK_IP: ${SUPERLINK_IP:-127.0.0.1} diff --git a/distributed/client/compose.yml b/distributed/client/compose.yml deleted file mode 100644 index 9a20f02f..00000000 --- a/distributed/client/compose.yml +++ /dev/null @@ -1,130 +0,0 @@ -services: - supernode-1: - image: flwr/supernode:${FLWR_VERSION:-1.16.0} - command: - - --superlink - - ${SUPERLINK_IP:-127.0.0.1}:9092 - - --clientappio-api-address - - 0.0.0.0:9094 - - --isolation - - process - - --node-config - - "partition-id=0 num-partitions=2" - - --root-certificates - - certificates/superlink-ca.crt - secrets: - - source: superlink-ca-certfile - target: /app/certificates/superlink-ca.crt - - supernode-2: - image: flwr/supernode:${FLWR_VERSION:-1.16.0} - command: - - --superlink - - ${SUPERLINK_IP:-127.0.0.1}:9092 - - --clientappio-api-address - - 0.0.0.0:9095 - - --isolation - - process - - --node-config - - "partition-id=1 num-partitions=2" - - --root-certificates - - certificates/superlink-ca.crt - secrets: - - source: superlink-ca-certfile - target: /app/certificates/superlink-ca.crt - - # uncomment to add another SuperNode - # - # supernode-3: - # image: flwr/supernode:${FLWR_VERSION:-1.16.0} - # command: - # - --superlink - # - ${SUPERLINK_IP:-127.0.0.1}:9092 - # - --clientappio-api-address - # - 0.0.0.0:9096 - # - --isolation - # - process - # - --node-config - # - "partition-id=1 num-partitions=2" - # - --root-certificates - # - certificates/superlink-ca.crt - # secrets: - # - source: superlink-ca-certfile - # target: /app/certificates/superlink-ca.crt - - clientapp-1: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-1:9094 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-1 - - clientapp-2: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-clientapp"] - command: - - --insecure - - --clientappio-api-address - - supernode-2:9095 - deploy: - resources: - limits: - cpus: "2" - stop_signal: SIGINT - depends_on: - - supernode-2 - # uncomment to add another ClientApp - # - # clientapp-3: - # build: - # context: ${PROJECT_DIR:-.} - # dockerfile_inline: | - # FROM flwr/clientapp:${FLWR_VERSION:-1.16.0} - - # WORKDIR /app - # COPY --chown=app:app pyproject.toml . - # RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - # && python -m pip install -U --no-cache-dir . - - # ENTRYPOINT ["flwr-clientapp"] - # command: - # - --insecure - # - --clientappio-api-address - # - supernode-3:9096 - # deploy: - # resources: - # limits: - # cpus: "2" - # stop_signal: SIGINT - # depends_on: - # - supernode-3 - -secrets: - superlink-ca-certfile: - file: ../superlink-certificates/ca.crt diff --git a/distributed/server/compose.yml b/distributed/server/compose.yml deleted file mode 100644 index 89f5de86..00000000 --- a/distributed/server/compose.yml +++ /dev/null @@ -1,49 +0,0 @@ -services: - superlink: - image: flwr/superlink:${FLWR_VERSION:-1.16.0} - command: - - --isolation - - process - - --ssl-ca-certfile=certificates/ca.crt - - --ssl-certfile=certificates/server.pem - - --ssl-keyfile=certificates/server.key - - --database=state/state.db - volumes: - - ./state/:/app/state/:rw - secrets: - - source: superlink-ca-certfile - target: /app/certificates/ca.crt - - source: superlink-certfile - target: /app/certificates/server.pem - - source: superlink-keyfile - target: /app/certificates/server.key - ports: - - 9092:9092 - - 9093:9093 - - serverapp: - build: - context: ${PROJECT_DIR:-.} - dockerfile_inline: | - FROM flwr/serverapp:${FLWR_VERSION:-1.16.0} - - WORKDIR /app - COPY --chown=app:app pyproject.toml . - RUN sed -i 's/.*flwr\[simulation\].*//' pyproject.toml \ - && python -m pip install -U --no-cache-dir . - - ENTRYPOINT ["flwr-serverapp"] - command: - - --insecure - - --serverappio-api-address - - superlink:9091 - depends_on: - - superlink - -secrets: - superlink-ca-certfile: - file: ../superlink-certificates/ca.crt - superlink-certfile: - file: ../superlink-certificates/server.pem - superlink-keyfile: - file: ../superlink-certificates/server.key From 2fbf911350369887ba6454a79411c0e9e335d02e Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Thu, 8 May 2025 17:31:51 +0530 Subject: [PATCH 13/18] added model misconfig fault --- aiopslab/generators/fault/inject_virtual.py | 15 +++- .../flower_model_misconfig/__init__.py | 6 ++ .../flower_model_misconfig/model_misconfig.py | 80 +++++++++++++++++++ aiopslab/orchestrator/problems/registry.py | 2 + aiopslab/service/shell.py | 2 +- clients/llama.py | 2 +- 6 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 aiopslab/orchestrator/problems/flower_model_misconfig/__init__.py create mode 100644 aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py diff --git a/aiopslab/generators/fault/inject_virtual.py b/aiopslab/generators/fault/inject_virtual.py index af9fafc6..2ee4089d 100644 --- a/aiopslab/generators/fault/inject_virtual.py +++ b/aiopslab/generators/fault/inject_virtual.py @@ -265,7 +265,20 @@ def recover_container_stop(self, microservices: list[str]): for service in microservices: self.docker.get_container(service).start() print(f"Started container {service}.") - + + def inject_model_misconfig(self, microservices: list[str]): + """Inject a fault to misconfigure the model in the Flower application.""" + for service in microservices: + command = f""" docker exec -it {service} sh -c "sed -i '24s/84/80/' /app/.flwr/apps/*/task.py" """ + self.docker.exec_command(command) + print(f"Changed model configuration for service: {service}") + + def recover_model_misconfig(self, microservices: list[str]): + for service in microservices: + command = f""" docker exec -it {service} sh -c "sed -i '24s/80/84/' /app/.flwr/apps/*/task.py" """ + self.docker.exec_command(command) + print(f"Recovered model configuration for service: {service}") + ############# HELPER FUNCTIONS ################ def _wait_for_pods_ready(self, microservices: list[str], timeout: int = 30): for service in microservices: diff --git a/aiopslab/orchestrator/problems/flower_model_misconfig/__init__.py b/aiopslab/orchestrator/problems/flower_model_misconfig/__init__.py new file mode 100644 index 00000000..1db81e08 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower_model_misconfig/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from .model_misconfig import ( + FlowerModelMisconfigDetection +) \ No newline at end of file diff --git a/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py b/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py new file mode 100644 index 00000000..62e0d8a9 --- /dev/null +++ b/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Model misconfiguration fault in the Flower application.""" + +import time +from typing import Any + +from aiopslab.orchestrator.tasks import * +from aiopslab.service.dock import Docker +from aiopslab.service.apps.flower import Flower +from aiopslab.paths import TARGET_MICROSERVICES +from aiopslab.session import SessionItem +from aiopslab.generators.fault.inject_virtual import VirtualizationFaultInjector + + +class FlowerModelMisconfigBaseTask: + def __init__(self, faulty_service: str = "user-service"): + self.app = Flower() + self.docker = Docker() + self.namespace = self.app.namespace + self.faulty_service = faulty_service + self.train_dir = TARGET_MICROSERVICES / "flower" + + def start_workload(self): + print("== Start Workload ==") + command = "flwr run train local-deployment" + self.docker.exec_command(command, cwd=self.train_dir) + + print("Waiting for workload to start...") + time.sleep(10) + + print("Injecting fault again...") + self.inject_fault() + + print("Waiting for faults to propagate...") + time.sleep(60) + print("Faults propagated.") + + def inject_fault(self): + print("== Fault Injection ==") + injector = VirtualizationFaultInjector(namespace=self.namespace) + injector._inject( + fault_type="model_misconfig", + microservices=[self.faulty_service], + ) + print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + + def recover_fault(self): + print("== Fault Recovery ==") + injector = VirtualizationFaultInjector(namespace=self.namespace) + injector._recover( + fault_type="model_misconfig", + microservices=[self.faulty_service], + ) + print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + + +################## Detection Problem ################## +class FlowerModelMisconfigDetection(FlowerModelMisconfigBaseTask, DetectionTask): + def __init__(self, faulty_service: str = "clientapp-1"): + FlowerModelMisconfigBaseTask.__init__(self, faulty_service=faulty_service) + DetectionTask.__init__(self, self.app) + + def eval(self, soln: Any, trace: list[SessionItem], duration: float): + print("== Evaluation ==") + expected_solution = "Yes" + + if isinstance(soln, str): + if soln.strip().lower() == expected_solution.lower(): + print(f"Correct detection: {soln}") + self.add_result("Detection Accuracy", "Correct") + else: + print(f"Incorrect detection: {soln}") + self.add_result("Detection Accuracy", "Incorrect") + else: + print("Invalid solution format") + self.add_result("Detection Accuracy", "Invalid Format") + + return super().eval(soln, trace, duration) diff --git a/aiopslab/orchestrator/problems/registry.py b/aiopslab/orchestrator/problems/registry.py index efdb9d45..9cf36e5e 100644 --- a/aiopslab/orchestrator/problems/registry.py +++ b/aiopslab/orchestrator/problems/registry.py @@ -28,6 +28,7 @@ from aiopslab.orchestrator.problems.wrong_bin_usage import * from aiopslab.orchestrator.problems.operator_misoperation import * from aiopslab.orchestrator.problems.flower_node_stop import * +from aiopslab.orchestrator.problems.flower_model_misconfig import * class ProblemRegistry: @@ -214,6 +215,7 @@ def __init__(self): # "operator_wrong_update_strategy-localization-1": K8SOperatorWrongUpdateStrategyLocalization, # Flower "flower_node_stop-detection": FlowerNodeStopDetection, + "flower_model_misconfig-detection": FlowerModelMisconfigDetection, } def get_problem_instance(self, problem_id: str): diff --git a/aiopslab/service/shell.py b/aiopslab/service/shell.py index 83d8fec8..af482bac 100644 --- a/aiopslab/service/shell.py +++ b/aiopslab/service/shell.py @@ -58,7 +58,7 @@ def local_exec(command: str, input_data=None, cwd=None): print(f"[ERROR] Command execution failed: {error_message}") return error_message else: - output_message = out.stdout.decode("utf-8") + output_message = out.stdout.decode("utf-8") + out.stderr.decode("utf-8") print(output_message) return output_message diff --git a/clients/llama.py b/clients/llama.py index c0e9c13b..372e9704 100644 --- a/clients/llama.py +++ b/clients/llama.py @@ -64,7 +64,7 @@ def _filter_dict(self, dictionary, filter_func): orchestrator = Orchestrator() orchestrator.register_agent(agent, name="llama-w-shell") - pid = "flower_node_stop-detection" + pid = "flower_model_misconfig-detection" problem_desc, instructs, apis = orchestrator.init_problem(pid) agent.init_context(problem_desc, instructs, apis) asyncio.run(orchestrator.start_problem(max_steps=10)) From 51da78a1c0dd52dc9f54dda4d0a2eba10ffc644d Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Thu, 22 May 2025 22:12:43 +0530 Subject: [PATCH 14/18] fast forwarded aiopslab-applications --- aiopslab-applications | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiopslab-applications b/aiopslab-applications index 5221ef96..231ccc32 160000 --- a/aiopslab-applications +++ b/aiopslab-applications @@ -1 +1 @@ -Subproject commit 5221ef962879546bb3c977c8a256ce117697b9f2 +Subproject commit 231ccc32d94b2e202cf11ba08be371d372c44b3d From a6bd7c900da700a3d26b265e5de5228ec672a01b Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Thu, 19 Jun 2025 22:28:50 +0530 Subject: [PATCH 15/18] removed sleep calls --- .../flower_model_misconfig/model_misconfig.py | 38 +++++++++++++------ clients/llama.py | 4 -- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py b/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py index 62e0d8a9..9495d716 100644 --- a/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py +++ b/aiopslab/orchestrator/problems/flower_model_misconfig/model_misconfig.py @@ -27,24 +27,40 @@ def start_workload(self): command = "flwr run train local-deployment" self.docker.exec_command(command, cwd=self.train_dir) + path = "/app/.flwr/apps" + check = f""" docker exec -it {self.faulty_service} sh -c "test -d {path} && echo 'exists'" """ + print("Waiting for workload to start...") - time.sleep(10) + while True: + exists = self.docker.exec_command(check) + if exists.strip() == "exists": + break + time.sleep(1) + print("Workload started successfully.") - print("Injecting fault again...") - self.inject_fault() + # Inject fault after workload starts, since the required files are created during the workload + print("Injecting fault...") + self.inject_fault(inject=True) print("Waiting for faults to propagate...") - time.sleep(60) + while True: + logs = self.docker.get_logs(self.faulty_service) + if "error" in logs.lower(): + break + time.sleep(1) print("Faults propagated.") - def inject_fault(self): + def inject_fault(self, inject: bool = False): print("== Fault Injection ==") - injector = VirtualizationFaultInjector(namespace=self.namespace) - injector._inject( - fault_type="model_misconfig", - microservices=[self.faulty_service], - ) - print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + if inject: + injector = VirtualizationFaultInjector(namespace=self.namespace) + injector._inject( + fault_type="model_misconfig", + microservices=[self.faulty_service], + ) + print(f"Service: {self.faulty_service} | Namespace: {self.namespace}\n") + else: + print("Fault injection skipped.") def recover_fault(self): print("== Fault Recovery ==") diff --git a/clients/llama.py b/clients/llama.py index 372e9704..a05795e4 100644 --- a/clients/llama.py +++ b/clients/llama.py @@ -2,10 +2,6 @@ Naive LLaMA client (with shell access) for AIOpsLab. """ -import sys -sys.path.append("..") -sys.path.append("../..") - import asyncio from aiopslab.orchestrator import Orchestrator From deb1478467b75469e83b34746a3b1566c3b25e01 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Fri, 20 Jun 2025 21:35:09 +0530 Subject: [PATCH 16/18] prevent openebs and prometheus launch for docker problems --- aiopslab/orchestrator/orchestrator.py | 3 ++- aiopslab/orchestrator/problems/registry.py | 9 +++++++++ clients/utils/llm.py | 3 --- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/aiopslab/orchestrator/orchestrator.py b/aiopslab/orchestrator/orchestrator.py index c29b68b1..cdf109f1 100644 --- a/aiopslab/orchestrator/orchestrator.py +++ b/aiopslab/orchestrator/orchestrator.py @@ -45,10 +45,11 @@ def init_problem(self, problem_id: str): self.session = Session() print(f"Session ID: {self.session.session_id}") prob = self.probs.get_problem_instance(problem_id) + deployment = self.probs.get_problem_deployment(problem_id) self.session.set_problem(prob, pid=problem_id) self.session.set_agent(self.agent_name) - if "flower" not in problem_id: # temporary fix for testing, will edit later + if deployment != "docker": print("Setting up OpenEBS...") # Install OpenEBS diff --git a/aiopslab/orchestrator/problems/registry.py b/aiopslab/orchestrator/problems/registry.py index 9cf36e5e..9a913227 100644 --- a/aiopslab/orchestrator/problems/registry.py +++ b/aiopslab/orchestrator/problems/registry.py @@ -217,6 +217,10 @@ def __init__(self): "flower_node_stop-detection": FlowerNodeStopDetection, "flower_model_misconfig-detection": FlowerModelMisconfigDetection, } + self.DOCKER_REGISTRY = [ + "flower_node_stop-detection", + "flower_model_misconfig-detection", + ] def get_problem_instance(self, problem_id: str): if problem_id not in self.PROBLEM_REGISTRY: @@ -236,3 +240,8 @@ def get_problem_count(self, task_type: str = None): if task_type: return len([k for k in self.PROBLEM_REGISTRY.keys() if task_type in k]) return len(self.PROBLEM_REGISTRY) + + def get_problem_deployment(self, problem_id: str): + if problem_id in self.DOCKER_REGISTRY: + return "docker" + return "k8s" diff --git a/clients/utils/llm.py b/clients/utils/llm.py index 09b5851c..5ecaebf2 100644 --- a/clients/utils/llm.py +++ b/clients/utils/llm.py @@ -266,9 +266,6 @@ def inference(self, payload: list[dict[str, str]]) -> list[str]: def run(self, payload: list[dict[str, str]]) -> list[str]: response = self.inference(payload) - - print("Llama response:", response) - if self.cache is not None: self.cache.add_to_cache(payload, response) self.cache.save_cache() From 16d469e0628c3483bad091d43a4b2759b20693a1 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Fri, 20 Jun 2025 21:35:50 +0530 Subject: [PATCH 17/18] updated requirements --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b97b2f0f..eeb5f7e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ python-dotenv = "^1.0.1" vllm = "^0.7.3" transformers = "^4.49.0" fastapi = "^0.115.12" +groq = "^0.28.0" +flwr = "^1.19.0" [build-system] From 0dd1473a931b00ade0f3b5b2d5902111985176d5 Mon Sep 17 00:00:00 2001 From: adityapgupta Date: Fri, 8 Aug 2025 00:01:19 +0530 Subject: [PATCH 18/18] updating aiopslab-applications --- aiopslab-applications | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiopslab-applications b/aiopslab-applications index 231ccc32..48e03edb 160000 --- a/aiopslab-applications +++ b/aiopslab-applications @@ -1 +1 @@ -Subproject commit 231ccc32d94b2e202cf11ba08be371d372c44b3d +Subproject commit 48e03edb4732468331b6963bc4644e8bae08fac1