diff --git a/teepod/rpc/proto/teepod_rpc.proto b/teepod/rpc/proto/teepod_rpc.proto index c5ac9ff2b..b90d9e2b9 100644 --- a/teepod/rpc/proto/teepod_rpc.proto +++ b/teepod/rpc/proto/teepod_rpc.proto @@ -91,6 +91,22 @@ message PublicKeyResponse { bytes public_key = 1; } +message GetInfoResponse { + bool found = 1; + optional VmInfo info = 2; +} + +message ResizeVmRequest { + // Unique identifier for the VM + string id = 1; + // Number of vCPUs + optional uint32 vcpu = 2; + // Memory in MB + optional uint32 memory = 3; + // Disk size in GB + optional uint32 disk_size = 4; +} + // Service definition for Teepod service Teepod { // RPC to create a VM @@ -111,4 +127,10 @@ service Teepod { // Get Env encrypt public key rpc GetAppEnvEncryptPubKey(AppId) returns (PublicKeyResponse); + + // Get VM info by ID + rpc GetInfo(Id) returns (GetInfoResponse); + + // RPC to resize a VM + rpc ResizeVm(ResizeVmRequest) returns (google.protobuf.Empty); } diff --git a/teepod/src/app.rs b/teepod/src/app.rs index ab256fbe3..1ce854b08 100644 --- a/teepod/src/app.rs +++ b/teepod/src/app.rs @@ -193,54 +193,13 @@ impl App { let mut infos = self .lock() .iter_vms() - .map(|vm| vm.merge_info(&vms, &self.work_dir(&vm.manifest.id))) + .map(|vm| vm.merge_info(vms.get(&vm.manifest.id), &self.work_dir(&vm.manifest.id))) .collect::>(); infos.sort_by(|a, b| a.manifest.created_at_ms.cmp(&b.manifest.created_at_ms)); let gw = &self.config.gateway; - let lst = infos - .into_iter() - .map(|info| pb::VmInfo { - id: info.manifest.id, - name: info.manifest.name.clone(), - status: info.status.to_string(), - uptime: info.uptime, - configuration: Some(pb::VmConfiguration { - name: info.manifest.name, - image: info.manifest.image, - compose_file: { - let workdir = VmWorkDir::new(&info.workdir); - fs::read_to_string(workdir.app_compose_path()).unwrap_or_default() - }, - encrypted_env: { - let workdir = VmWorkDir::new(&info.workdir); - fs::read(workdir.encrypted_env_path()).unwrap_or_default() - }, - vcpu: info.manifest.vcpu, - memory: info.manifest.memory, - disk_size: info.manifest.disk_size, - ports: info - .manifest - .port_map - .into_iter() - .map(|pm| pb::PortMapping { - protocol: pm.protocol.as_str().into(), - host_port: pm.from as u32, - vm_port: pm.to as u32, - }) - .collect(), - }), - app_url: info.instance_id.as_ref().map(|id| { - format!( - "https://{id}-{}.{}:{}", - gw.tappd_port, gw.base_domain, gw.port - ) - }), - app_id: info.manifest.app_id, - instance_id: info.instance_id, - }) - .collect(); + let lst = infos.into_iter().map(|info| info.to_pb(gw)).collect(); Ok(lst) } @@ -260,6 +219,17 @@ impl App { }) .collect()) } + + pub async fn get_vm(&self, id: &str) -> Result> { + let proc_state = self.supervisor.info(id).await?; + let Some(cfg) = self.lock().get(id) else { + return Ok(None); + }; + let info = cfg + .merge_info(proc_state.as_ref(), &self.work_dir(id)) + .to_pb(&self.config.gateway); + Ok(Some(info)) + } } pub(crate) struct AppState { diff --git a/teepod/src/app/qemu.rs b/teepod/src/app/qemu.rs index c557fa4f0..59ee859db 100644 --- a/teepod/src/app/qemu.rs +++ b/teepod/src/app/qemu.rs @@ -1,20 +1,22 @@ //! QEMU related code +use crate::{ + app::Manifest, + config::{GatewayConfig, Networking}, +}; use std::{ - collections::HashMap, ops::Deref, path::{Path, PathBuf}, process::Command, time::{Duration, SystemTime}, }; -use crate::{app::Manifest, config::Networking}; - use super::image::Image; use anyhow::{Context, Result}; use bon::Builder; use fs_err as fs; use serde::{Deserialize, Serialize}; use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; +use teepod_rpc as pb; #[derive(Debug, Deserialize)] pub struct InstanceInfo { @@ -68,13 +70,53 @@ fn create_hd( Ok(()) } +impl VmInfo { + pub fn to_pb(&self, gw: &GatewayConfig) -> pb::VmInfo { + let workdir = VmWorkDir::new(&self.workdir); + pb::VmInfo { + id: self.manifest.id.as_str().into(), + name: self.manifest.name.as_str().into(), + status: self.status.into(), + uptime: self.uptime.as_str().into(), + configuration: Some(pb::VmConfiguration { + name: self.manifest.name.as_str().into(), + image: self.manifest.image.as_str().into(), + compose_file: { + fs::read_to_string(workdir.app_compose_path()).unwrap_or_default() + }, + encrypted_env: { fs::read(workdir.encrypted_env_path()).unwrap_or_default() }, + vcpu: self.manifest.vcpu, + memory: self.manifest.memory, + disk_size: self.manifest.disk_size, + ports: self + .manifest + .port_map + .iter() + .map(|pm| pb::PortMapping { + protocol: pm.protocol.as_str().into(), + host_port: pm.from as u32, + vm_port: pm.to as u32, + }) + .collect(), + }), + app_url: self.instance_id.as_ref().map(|id| { + format!( + "https://{id}-{}.{}:{}", + gw.tappd_port, gw.base_domain, gw.port + ) + }), + app_id: self.manifest.app_id.as_str().into(), + instance_id: self.instance_id.as_deref().map(Into::into), + } + } +} + impl VmConfig { - pub fn merge_info(&self, states: &HashMap, workdir: &VmWorkDir) -> VmInfo { + pub fn merge_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { fn truncate(d: Duration) -> Duration { Duration::from_secs(d.as_secs()) } - let info = states.get(&self.manifest.id); - let is_running = match &info { + let is_running = match proc_state { Some(info) => info.state.status.is_running(), None => false, }; @@ -96,8 +138,8 @@ impl VmConfig { } } } - let uptime = display_ts(info.and_then(|info| info.state.started_at.as_ref())); - let exited_at = display_ts(info.and_then(|info| info.state.stopped_at.as_ref())); + let uptime = display_ts(proc_state.and_then(|info| info.state.started_at.as_ref())); + let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref())); let instance_id = workdir.instance_info().ok().map(|info| info.instance_id); VmInfo { manifest: self.manifest.clone(), diff --git a/teepod/src/main_service.rs b/teepod/src/main_service.rs index 9c8990317..af66dec1d 100644 --- a/teepod/src/main_service.rs +++ b/teepod/src/main_service.rs @@ -8,8 +8,8 @@ use ra_rpc::client::RaClient; use ra_rpc::{Attestation, RpcCall}; use teepod_rpc::teepod_server::{TeepodRpc, TeepodServer}; use teepod_rpc::{ - AppId, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, StatusResponse, - UpgradeAppRequest, VmConfiguration, + AppId, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, + ResizeVmRequest, StatusResponse, UpgradeAppRequest, VmConfiguration, }; use tracing::warn; @@ -252,6 +252,55 @@ impl TeepodRpc for RpcHandler { public_key: response.public_key, }) } + + async fn get_info(self, request: Id) -> Result { + if let Some(vm) = self.app.get_vm(&request.id).await? { + Ok(GetInfoResponse { + found: true, + info: Some(vm), + }) + } else { + Ok(GetInfoResponse { + found: false, + info: None, + }) + } + } + + async fn resize_vm(self, request: ResizeVmRequest) -> Result<()> { + let vm = self + .app + .get_vm(&request.id) + .await? + .ok_or_else(|| anyhow::anyhow!("vm not found: {}", request.id))?; + if vm.status != "stopped" { + return Err(anyhow::anyhow!( + "vm should be stopped before resize: {}", + request.id + )); + } + let work_dir = self.app.config.run_path.join(&request.id); + let vm_work_dir = VmWorkDir::new(&work_dir); + let mut manifest = vm_work_dir.manifest().context("failed to read manifest")?; + if let Some(vcpu) = request.vcpu { + manifest.vcpu = vcpu; + } + if let Some(memory) = request.memory { + manifest.memory = memory; + } + if let Some(disk_size) = request.disk_size { + // it only updates the manifesta and does NOT affect the real storage alloc at this time. + manifest.disk_size = disk_size; + } + vm_work_dir + .put_manifest(&manifest) + .context("failed to update manifest")?; + self.app + .load_vm(work_dir) + .await + .context("Failed to load VM")?; + Ok(()) + } } impl RpcCall for RpcHandler { diff --git a/teepod/src/vm.rs b/teepod/src/vm.rs deleted file mode 100644 index 215e5cd75..000000000 --- a/teepod/src/vm.rs +++ /dev/null @@ -1,294 +0,0 @@ -pub(crate) mod image { - use fs_err as fs; - use std::path::{Path, PathBuf}; - - use anyhow::{bail, Context, Result}; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Serialize, Deserialize)] - pub struct ImageInfo { - pub cmdline: Option, - pub kernel: String, - pub initrd: String, - pub hda: Option, - pub rootfs: Option, - pub bios: Option, - pub rootfs_hash: Option, - } - - impl ImageInfo { - pub fn load(filename: PathBuf) -> Result { - let file = fs::File::open(filename).context("failed to open image info")?; - let info: ImageInfo = - serde_json::from_reader(file).context("failed to parse image info")?; - Ok(info) - } - } - - #[derive(Debug)] - pub struct Image { - pub info: ImageInfo, - pub initrd: PathBuf, - pub kernel: PathBuf, - pub hda: Option, - pub rootfs: Option, - pub bios: Option, - } - - impl Image { - pub fn load(base_path: impl AsRef) -> Result { - let base_path = fs::canonicalize(base_path.as_ref())?; - let info = ImageInfo::load(base_path.join("metadata.json"))?; - let initrd = base_path.join(&info.initrd); - let kernel = base_path.join(&info.kernel); - let hda = info.hda.as_ref().map(|hda| base_path.join(hda)); - let rootfs = info.rootfs.as_ref().map(|rootfs| base_path.join(rootfs)); - let bios = info.bios.as_ref().map(|bios| base_path.join(bios)); - Self { - info, - hda, - initrd, - kernel, - rootfs, - bios, - } - .ensure_exists() - } - - fn ensure_exists(self) -> Result { - if !self.initrd.exists() { - bail!("Initrd does not exist: {}", self.initrd.display()); - } - if !self.kernel.exists() { - bail!("Kernel does not exist: {}", self.kernel.display()); - } - if let Some(hda) = &self.hda { - if !hda.exists() { - bail!("Hda does not exist: {}", hda.display()); - } - } - if let Some(rootfs) = &self.rootfs { - if !rootfs.exists() { - bail!("Rootfs does not exist: {}", rootfs.display()); - } - } - if let Some(bios) = &self.bios { - if !bios.exists() { - bail!("Bios does not exist: {}", bios.display()); - } - } - Ok(self) - } - } -} - -mod qemu { - //! QEMU related code - use std::{ - collections::HashMap, - path::{Path, PathBuf}, - process::Command, - sync::Arc, - }; - - use crate::{ - app::{Manifest, VmWorkDir}, - config::Networking, - }; - - use super::image::Image; - use anyhow::Result; - use bon::Builder; - use fs_err as fs; - use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; - - #[derive(Debug, Deserialize)] - pub struct InstanceInfo { - pub instance_id: String, - } - - pub struct VmInfo { - pub manifest: Manifest, - pub workdir: PathBuf, - pub status: &'static str, - pub uptime: String, - pub exited_at: Option, - pub instance_id: Option, - } - - #[derive(Debug)] - pub struct TdxConfig { - /// Guest CID for vhost-vsock - pub cid: u32, - } - - #[derive(Debug, Builder)] - pub struct VmConfig { - pub manifest: Manifest, - pub image: Image, - pub tdx_config: Option, - pub networking: Networking, - } - - fn create_hd( - image_file: impl AsRef, - backing_file: Option>, - size: &str, - ) -> Result<()> { - let mut command = Command::new("qemu-img"); - command.arg("create").arg("-f").arg("qcow2"); - if let Some(backing_file) = backing_file { - command - .arg("-o") - .arg(format!("backing_file={}", backing_file.as_ref().display())); - command.arg("-o").arg("backing_fmt=qcow2"); - } - command.arg(image_file.as_ref()); - command.arg(size); - command.spawn()?.wait()?; - Ok(()) - } - - impl VmConfig { - pub fn merge_info( - &self, - states: &HashMap, - instance_id: Option, - ) -> VmInfo { - fn truncate(d: Duration) -> Duration { - Duration::from_secs(d.as_secs()) - } - let info = states.get(&self.config.manifest.id); - let is_running = match &info { - Some(info) => info.state.status.is_running(), - None => false, - }; - let status = match (self.started, is_running) { - (true, true) => "running", - (true, false) => "exited", - (false, true) => "stopping", - (false, false) => "stopped", - }; - - fn display_ts(t: Option<&SystemTime>) -> String { - match t { - None => "never".into(), - Some(t) => { - let ts = t.duration_since(UNIX_EPOCH).unwrap_or(Duration::MAX); - humantime::format_duration(truncate(ts)).to_string() - } - } - } - let uptime = display_ts(info.and_then(|info| info.state.started_at.as_ref())); - let exited_at = display_ts(info.and_then(|info| info.state.stopped_at.as_ref())); - - VmInfo { - manifest: self.config.manifest.clone(), - workdir: self.workdir.clone(), - instance_id, - status, - uptime, - exited_at: Some(exited_at), - } - } - - pub fn config_qemu(&self, qemu: &Path, workdir: impl AsRef) -> Result { - let workdir = VmWorkDir::new(workdir); - let serial_file = workdir.serial_file(); - let shared_dir = workdir.shared_dir(); - let disk_size = format!("{}G", self.manifest.disk_size); - let hda_path = workdir.hda_path(); - if !hda_path.exists() { - create_hd(&hda_path, self.image.hda.as_ref(), &disk_size)?; - } - if !shared_dir.exists() { - fs::create_dir_all(&shared_dir)?; - } - let mut command = Command::new(qemu); - command.arg("-accel").arg("kvm"); - command.arg("-cpu").arg("host"); - command.arg("-smp").arg(self.manifest.vcpu.to_string()); - command.arg("-m").arg(format!("{}M", self.manifest.memory)); - command.arg("-nographic"); - command.arg("-nodefaults"); - command - .arg("-serial") - .arg(format!("file:{}", serial_file.display())); - command.arg("-kernel").arg(&self.image.kernel); - command.arg("-initrd").arg(&self.image.initrd); - command - .arg("-drive") - .arg(format!("file={},if=none,id=hd0", hda_path.display())) - .arg("-device") - .arg(format!("virtio-blk-pci,drive=hd0")); - if let Some(rootfs) = &self.image.rootfs { - command.arg("-cdrom").arg(rootfs); - } - if let Some(bios) = &self.image.bios { - command.arg("-bios").arg(bios); - } - let netdev = match &self.networking { - Networking::User(netcfg) => { - let mut netdev = format!( - "user,id=net0,net={},dhcpstart={},restrict={}", - netcfg.net, - netcfg.dhcp_start, - if netcfg.restrict { "yes" } else { "no" } - ); - for pm in &self.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - pm.protocol.as_str(), - pm.address, - pm.from, - pm.to - )); - } - netdev - } - Networking::Custom(netcfg) => netcfg.netdev.clone(), - }; - command.arg("-netdev").arg(netdev); - command.arg("-device").arg("virtio-net-pci,netdev=net0"); - if let Some(tdx) = &self.tdx_config { - command - .arg("-machine") - .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); - command.arg("-object").arg("tdx-guest,id=tdx"); - command - .arg("-device") - .arg(format!("vhost-vsock-pci,guest-cid={}", tdx.cid)); - } - command.arg("-virtfs").arg(format!( - "local,path={},mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0", - shared_dir.display() - )); - if let Some(cmdline) = &self.image.info.cmdline { - command.arg("-append").arg(cmdline); - } - - let args = command - .get_args() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - let pidfile_path = workdir.pid_file(); - let stdout_path = workdir.stdout_file(); - let stderr_path = workdir.stderr_file(); - - let workdir = workdir.path(); - let process_config = ProcessConfig { - id: self.manifest.id.clone(), - args, - name: self.manifest.name.clone(), - command: qemu.to_string_lossy().to_string(), - env: Default::default(), - cwd: workdir.to_string_lossy().to_string(), - stdout: stdout_path.to_string_lossy().to_string(), - stderr: stderr_path.to_string_lossy().to_string(), - pidfile: pidfile_path.to_string_lossy().to_string(), - }; - Ok(process_config) - } - } -} diff --git a/tproxy/rpc/proto/tproxy_rpc.proto b/tproxy/rpc/proto/tproxy_rpc.proto index f42b2ab07..f4eb4e266 100644 --- a/tproxy/rpc/proto/tproxy_rpc.proto +++ b/tproxy/rpc/proto/tproxy_rpc.proto @@ -71,6 +71,16 @@ message AcmeInfoResponse { repeated bytes hist_keys = 2; } +// Get HostInfo for associated instance id. +message GetInfoRequest { + string id = 1; +} + +message GetInfoResponse { + bool found = 1; + optional HostInfo info = 2; +} + service Tproxy { // Register a new proxied CVM. rpc RegisterCvm(RegisterCvmRequest) returns (RegisterCvmResponse) {} @@ -78,4 +88,6 @@ service Tproxy { rpc List(google.protobuf.Empty) returns (ListResponse) {} // List all ACME account URIs and the public key history of the certificates for the Content Addressable HTTPS. rpc AcmeInfo(google.protobuf.Empty) returns (AcmeInfoResponse) {} + // Find Proxied HostInfo by instance ID + rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {} } diff --git a/tproxy/src/main_service.rs b/tproxy/src/main_service.rs index 3f54723ab..3e8f238fe 100644 --- a/tproxy/src/main_service.rs +++ b/tproxy/src/main_service.rs @@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize}; use tproxy_rpc::{ tproxy_server::{TproxyRpc, TproxyServer}, AcmeInfoResponse, HostInfo as PbHostInfo, ListResponse, RegisterCvmRequest, - RegisterCvmResponse, TappdConfig, WireGuardConfig, + RegisterCvmResponse, TappdConfig, WireGuardConfig, GetInfoRequest, + GetInfoResponse, }; use tracing::{debug, error, info}; @@ -368,6 +369,38 @@ impl TproxyRpc for RpcHandler { Ok(ListResponse { hosts }) } + async fn get_info(self, request: GetInfoRequest) -> Result { + let state = self.state.lock(); + let base_domain = &state.config.proxy.base_domain; + let handshakes = state.latest_handshakes(None)?; + + if let Some(instance) = state.state.instances.get(&request.id) { + let host_info = PbHostInfo { + id: instance.id.clone(), + ip: instance.ip.to_string(), + app_id: instance.app_id.clone(), + base_domain: base_domain.clone(), + port: state.config.proxy.listen_port as u32, + latest_handshake: { + let (ts, _) = handshakes + .get(&instance.public_key) + .copied() + .unwrap_or_default(); + ts + }, + }; + Ok(GetInfoResponse { + found: true, + info: Some(host_info), + }) + } else { + Ok(GetInfoResponse { + found: false, + info: None, + }) + } + } + async fn acme_info(self) -> Result { let state = self.state.lock(); let workdir = WorkDir::new(&state.config.certbot.workdir);