flasharray/kvm/adaptive: NVMe-TCP transport for FlashArray primary storage

PendingReleaseNotes

@@ -39,3 +39,15 @@ example.ver.1 > example.ver.2:
  which can now be attached to Instances. This is to prevent the Secondary
  Storage to grow to enormous sizes as Linux Distributions keep growing in
  size while a stripped down Linux should fit on a 2.88MB floppy.
+
+
+4.22.0 > 4.23.0:
+ * Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework
+ and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary
+ pool can now be delivered to KVM hypervisors over NVMe-TCP instead of
+ Fibre Channel by setting transport=nvme-tcp on the pool's provider URL.
+ Volumes are identified on the host via EUI-128 NGUIDs and attached to
+ guests as plain block devices through the native NVMe multipath layer;
+ no device-mapper multipath configuration is required. A new
+ Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase /
+ NVMeTCPAdapter on the KVM side back the new pool type.

api/src/main/java/com/cloud/storage/Storage.java

@@ -185,7 +185,8 @@ public static enum StoragePoolType {
  Linstor(true, true, EncryptionSupport.Storage),
  DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters
  StorPool(true, true, EncryptionSupport.Hypervisor),
- FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+ FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+ NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.<eui>)
 
  private final boolean shared;
  private final boolean overProvisioning;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -415,7 +415,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
  StoragePoolType.PowerFlex,
  StoragePoolType.Linstor,
  StoragePoolType.FiberChannel,
- StoragePoolType.CLVM).contains(primaryPool.getType())) {
+ StoragePoolType.CLVM,
+ StoragePoolType.NVMeTCP).contains(primaryPool.getType())) {
  newTemplate.setFormat(ImageFormat.RAW);
  } else {
  newTemplate.setFormat(ImageFormat.QCOW2);
@@ -448,7 +449,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
 
  public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map<String, String> details) {
  String path = null;
- if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) {
+ if (primaryStore.getPoolType() == StoragePoolType.FiberChannel
+ || primaryStore.getPoolType() == StoragePoolType.NVMeTCP) {
  path = destData.getPath();
  } else {
  path = details != null ? details.get("managedStoreTarget") : null;
@@ -3431,7 +3433,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) {
  StoragePoolType.PowerFlex,
  StoragePoolType.Linstor,
  StoragePoolType.FiberChannel,
- StoragePoolType.CLVM).contains(poolType)) {
+ StoragePoolType.CLVM,
+ StoragePoolType.NVMeTCP).contains(poolType)) {
  return ImageFormat.RAW;
  } else {
  return ImageFormat.QCOW2;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java

@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.storage;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
+import org.joda.time.Duration;
+
+import com.cloud.agent.api.to.HostTO;
+import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ProvisioningType;
+
+/**
+ * KVMStoragePool for NVMe-over-Fabrics pools. Mirror of
+ * {@link MultipathSCSIPool} for adapters based on
+ * {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated
+ * back to the adapter; the pool itself only tracks addressing/identity.
+ */
+public class MultipathNVMeOFPool implements KVMStoragePool {
+ private final String uuid;
+ private final String sourceHost;
+ private final int sourcePort;
+ private final String sourceDir;
+ private final Storage.StoragePoolType storagePoolType;
+ private final StorageAdaptor storageAdaptor;
+ private final Map<String, String> details;
+ private long capacity;
+ private long used;
+ private long available;
+
+ public MultipathNVMeOFPool(String uuid, String host, int port, String path,
+ Storage.StoragePoolType poolType, Map<String, String> poolDetails, StorageAdaptor adaptor) {
+ this.uuid = uuid;
+ this.sourceHost = host;
+ this.sourcePort = port;
+ this.sourceDir = path;
+ this.storagePoolType = poolType;
+ this.storageAdaptor = adaptor;
+ this.details = poolDetails;
+ this.capacity = 0;
+ this.used = 0;
+ this.available = 0;
+ }
+
+ public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) {
+ this.uuid = uuid;
+ this.sourceHost = null;
+ this.sourcePort = -1;
+ this.sourceDir = null;
+ this.storagePoolType = Storage.StoragePoolType.NVMeTCP;
+ this.storageAdaptor = adaptor;
+ this.details = new HashMap<>();
+ this.capacity = 0;
+ this.used = 0;
+ this.available = 0;
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) {
+ return null;
+ }
+
+ @Override
+ public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) {
+ return null;
+ }
+
+ @Override
+ public boolean connectPhysicalDisk(String volumeUuid, Map<String, String> details) {
+ return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false);
+ }
+
+ @Override
+ public KVMPhysicalDisk getPhysicalDisk(String volumeId) {
+ return storageAdaptor.getPhysicalDisk(volumeId, this);
+ }
+
+ @Override
+ public boolean disconnectPhysicalDisk(String volumeUuid) {
+ return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this);
+ }
+
+ @Override
+ public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) {
+ return true;
+ }
+
+ @Override
+ public List<KVMPhysicalDisk> listPhysicalDisks() {
+ return null;
+ }
+
+ @Override
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void setCapacity(long capacity) { this.capacity = capacity; }
+ @Override public long getCapacity() { return this.capacity; }
+ public void setUsed(long used) { this.used = used; }
+ @Override public long getUsed() { return this.used; }
+ public void setAvailable(long available) { this.available = available; }
+ @Override public long getAvailable() { return this.available; }
+
+ @Override public boolean refresh() { return false; }
+ @Override public boolean isExternalSnapshot() { return true; }
+ @Override public String getLocalPath() { return null; }
+ @Override public String getSourceHost() { return this.sourceHost; }
+ @Override public String getSourceDir() { return this.sourceDir; }
+ @Override public int getSourcePort() { return this.sourcePort; }
+ @Override public String getAuthUserName() { return null; }
+ @Override public String getAuthSecret() { return null; }
+ @Override public Storage.StoragePoolType getType() { return storagePoolType; }
+ @Override public boolean delete() { return false; }
+ @Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; }
+ @Override public boolean createFolder(String path) { return false; }
+ @Override public boolean supportsConfigDriveIso() { return false; }
+ @Override public Map<String, String> getDetails() { return this.details; }
+ @Override public boolean isPoolSupportHA() { return false; }
+ @Override public String getHearthBeatPath() { return null; }
+
+ @Override
+ public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) {
+ return null;
+ }
+
+ @Override public String getStorageNodeId() { return null; }
+
+ @Override
+ public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; }
+
+ @Override
+ public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout,
+ String volumeUUIDListString, String vmActivityCheckPath, long duration) {
+ return null;
+ }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.storage;
+
+import com.cloud.storage.Storage;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type.
+ * All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this
+ * class just binds that logic to a pool type so
+ * {@link KVMStoragePoolManager} can find it via reflection.
+ */
+public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase {
+ private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class);
+
+ public NVMeTCPAdapter() {
+ LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer");
+ }
+
+ @Override
+ public String getName() {
+ return "NVMeTCPAdapter";
+ }
+
+ @Override
+ public Storage.StoragePoolType getStoragePoolType() {
+ return Storage.StoragePoolType.NVMeTCP;
+ }
+
+ @Override
+ public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) {
+ return Storage.StoragePoolType.NVMeTCP.equals(type);
+ }
+}

plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java

@@ -35,6 +35,7 @@ public interface ProviderVolume {
  public String getExternalName();
  public String getExternalConnectionId();
  public enum AddressType {
- FIBERWWN
+ FIBERWWN,
+ NVMETCP
  }
 }

plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java

@@ -179,7 +179,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
  parameters.setHost(uri.getHost());
  parameters.setPort(uri.getPort());
  parameters.setPath(uri.getPath() + "?" + uri.getQuery());
- parameters.setType(StoragePoolType.FiberChannel);
+ parameters.setType(pickPoolType(uri));
  parameters.setZoneId(zoneId);
  parameters.setPodId(podId);
  parameters.setClusterId(clusterId);
@@ -402,4 +402,26 @@ public void disableStoragePool(DataStore store) {
  logger.info("Disabling storage pool {}", store);
  _dataStoreHelper.disable(store);
  }
+
+ /**
+ * Resolve the CloudStack StoragePoolType from the provider URL. Adaptive
+ * plugins advertise the underlying fabric via a {@code transport=} query
+ * parameter on the URL; when absent we keep the legacy FiberChannel
+ * default for backwards compatibility with adapters that still assume it.
+ */
+ private static StoragePoolType pickPoolType(java.net.URL uri) {
+ String query = uri.getQuery();
+ if (query != null) {
+ for (String tok : query.split("&")) {
+ int i = tok.indexOf('=');
+ if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) {
+ String value = tok.substring(i + 1);
+ if ("nvme-tcp".equalsIgnoreCase(value)) {
+ return StoragePoolType.NVMeTCP;
+ }
+ }
+ }
+ }
+ return StoragePoolType.FiberChannel;
+ }
 }