Upgrade to JRE17 and Upgrade System VMs/VRs to Python3 and Debian 12

INSTALL.md

@@ -15,7 +15,7 @@ was tested against a CentOS 7 x86_64 setup.
 
 Install tools and dependencies used for development:
 
- # yum -y install git java-11-openjdk java-11-openjdk-devel \
+ # yum -y install git java-17-openjdk java-17-openjdk-devel \
  mysql mysql-server mkisofs git gcc python MySQL-python openssh-clients wget
 
 Set up Maven (3.6.0):

debian/control

@@ -17,14 +17,14 @@ Description: A common package which contains files which are shared by several C
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), net-tools, sudo, python3-mysql.connector, augeas-tools, mysql-client | mariadb-client, adduser, bzip2, ipmitool, file, gawk, iproute2, qemu-utils, rng-tools, python3-dnspython, lsb-release, init-system-helpers (>= 1.14~), python3-setuptools
+Depends: ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), net-tools, sudo, python3-mysql.connector, augeas-tools, mysql-client | mariadb-client, adduser, bzip2, ipmitool, file, gawk, iproute2, qemu-utils, rng-tools, python3-dnspython, lsb-release, init-system-helpers (>= 1.14~), python3-setuptools
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent
@@ -34,7 +34,7 @@ Description: CloudStack agent
 
 Package: cloudstack-usage
 Architecture: all
-Depends: openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), init-system-helpers
+Depends: openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), init-system-helpers
 Description: CloudStack usage monitor
  The CloudStack usage monitor provides usage accounting across the entire cloud for
  cloud operators to charge based on usage parameters.

engine/api/pom.xml

@@ -71,7 +71,7 @@
  <dependency>
  <groupId>com.sun.xml.bind</groupId>
  <artifactId>jaxb-impl</artifactId>
- <version>${cs.jaxb.version}</version>
+ <version>${cs.jaxb.impl.version}</version>
  </dependency>
  </dependencies>
 </project>

engine/schema/pom.xml

@@ -101,8 +101,10 @@
  for (template in templateList) {
  def data = lines.findAll { it.contains(template) }
  if (data != null) {
- def hypervisor = template.tokenize('-')[-1]
- pom.properties["$hypervisor" + ".checksum"] = data[0].tokenize(' ')[0]
+ if (data.size() > 0) {
+ def hypervisor = template.tokenize('-')[-1]
+ pom.properties["$hypervisor" + ".checksum"] = data[0].tokenize(' ')[0]
+ }
  }
  }
  </source>

engine/schema/src/main/resources/META-INF/db/schema-41900to42000.sql

@@ -28,3 +28,9 @@ DROP INDEX `i_resource_count__type_accountId`,
 DROP INDEX `i_resource_count__type_domaintId`,
 ADD UNIQUE INDEX `i_resource_count__type_tag_accountId` (`type`,`tag`,`account_id`),
 ADD UNIQUE INDEX `i_resource_count__type_tag_domaintId` (`type`,`tag`,`domain_id`);
+
+-- Update Default System offering for Router to 512MiB
+UPDATE `cloud`.`service_offering` SET ram_size = 512 WHERE unique_name IN ("Cloud.Com-SoftwareRouter", "Cloud.Com-SoftwareRouter-Local",
+ "Cloud.Com-InternalLBVm", "Cloud.Com-InternalLBVm-Local",
+ "Cloud.Com-ElasticLBVm", "Cloud.Com-ElasticLBVm-Local")
+ AND system_use = 1 AND ram_size < 512;

framework/rest/pom.xml

@@ -68,7 +68,7 @@
  <dependency>
  <groupId>com.sun.xml.bind</groupId>
  <artifactId>jaxb-impl</artifactId>
- <version>${cs.jaxb.version}</version>
+ <version>${cs.jaxb.impl.version}</version>
  </dependency>
  <dependency>
  <groupId>org.apache.cxf</groupId>

packaging/centos8/cloud.spec

@@ -52,7 +52,7 @@ intelligent IaaS cloud implementation.
 
 %package management
 Summary: CloudStack management server UI
-Requires: java-11-openjdk
+Requires: java-17-openjdk
 Requires: (tzdata-java or timezone-java)
 Requires: python3
 Requires: bash
@@ -98,7 +98,7 @@ The Apache CloudStack files shared between agent and management server
 %package agent
 Summary: CloudStack Agent for KVM hypervisors
 Requires: (openssh-clients or openssh)
-Requires: java-11-openjdk
+Requires: java-17-openjdk
 Requires: tzdata-java
 Requires: %{name}-common = %{_ver}
 Requires: libvirt
@@ -135,7 +135,7 @@ The CloudStack baremetal agent
 
 %package usage
 Summary: CloudStack Usage calculation server
-Requires: java-11-openjdk
+Requires: java-17-openjdk
 Requires: tzdata-java
 Group: System Environment/Libraries
 %description usage
@@ -556,8 +556,8 @@ if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];
 fi
 
 %post marvin
-pip install --upgrade https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl
-pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
+pip3 install --upgrade https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl
+pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 
 #No default permission as the permission setup is complex
 %files management

packaging/systemd/cloudstack-management.default

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Dcom.sun.management.jmxremote=false -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err "
+JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Dcom.sun.management.jmxremote=false -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED"
 
 CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-mysql-ha/lib/*"
 
@@ -24,7 +24,7 @@ BOOTSTRAP_CLASS=org.apache.cloudstack.ServerDaemon
 ################################################################################################
 #You can uncomment one of these options if you want to enable Java remote debugging. #
 #You can change the parameters at your will. The 'address' field defines the port to be used. #
-################################################################################################
+################################################################################################
 # This option here should be used with 'systemmd' based operating systems such as CentOS7, Ubuntu 16, and so on.
 #JAVA_DEBUG="-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n"
 

packaging/systemd/cloudstack-usage.default

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-JAVA_OPTS="-Xms256m -Xmx2048m"
+JAVA_OPTS="-Xms256m -Xmx2048m --add-opens=java.base/java.lang=ALL-UNNAMED"
 
 CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar"
 

plugins/hypervisors/baremetal/pom.xml

@@ -45,7 +45,7 @@
  <dependency>
  <groupId>com.sun.xml.bind</groupId>
  <artifactId>jaxb-impl</artifactId>
- <version>${cs.jaxb.version}</version>
+ <version>${cs.jaxb.impl.version}</version>
  </dependency>
  </dependencies>
 </project>

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPatchSystemVmCommandWrapper.java

@@ -77,6 +77,7 @@ public Answer execute(PatchSystemVmCommand cmd, LibvirtComputingResource serverR
  if (patchResult.first()) {
  String scriptVersion = lines[1];
  if (StringUtils.isNotEmpty(patchResult.second())) {
+ logger.debug("Patch result of systemVM {}: {}", sysVMName, patchResult.second());
  String res = patchResult.second().replace("\n", " ");
  String[] output = res.split(":");
  if (output.length != 2) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java

@@ -102,7 +102,9 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo
  if (template != null) {
  response.setIsoId(template.getUuid());
  response.setIsoName(template.getName());
- response.setIsoState(template.getState().toString());
+ if (template.getState() != null) {
+ response.setIsoState(template.getState().toString());
+ }
  response.setDirectDownload(template.isDirectDownload());
  }
  response.setCreated(kubernetesSupportedVersion.getCreated());

plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManager.java

@@ -32,7 +32,7 @@
 
 public interface InternalLoadBalancerVMManager {
  //RAM/CPU for the system offering used by Internal LB VMs
- public static final int DEFAULT_INTERNALLB_VM_RAMSIZE = 256; // 256 MB
+ public static final int DEFAULT_INTERNALLB_VM_RAMSIZE = 512; // 512 MB
  public static final int DEFAULT_INTERNALLB_VM_CPU_MHZ = 256; // 256 MHz
 
  /**

pom.xml

@@ -50,7 +50,7 @@
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
  <project.systemvm.template.location>https://download.cloudstack.org/systemvm</project.systemvm.template.location>
- <project.systemvm.template.version>4.19.0.0</project.systemvm.template.version>
+ <project.systemvm.template.version>4.20.0.0</project.systemvm.template.version>
  <sonar.organization>apache</sonar.organization>
  <sonar.host.url>https://sonarcloud.io</sonar.host.url>
 
@@ -151,7 +151,9 @@
  <cs.maven-javadoc-plugin.version>3.1.1</cs.maven-javadoc-plugin.version>
  <cs.javax.annotation.version>1.3.2</cs.javax.annotation.version>
  <cs.jaxb.version>2.3.0</cs.jaxb.version>
- <cs.jaxws.version>2.3.2-1</cs.jaxws.version>
+ <cs.jaxb.impl.version>2.3.9</cs.jaxb.impl.version>
+ <cs.jakarta.xml.bind.version>2.3.3</cs.jakarta.xml.bind.version>
+ <cs.jaxws.version>2.3.7</cs.jaxws.version>
  <cs.jersey-client.version>2.26</cs.jersey-client.version>
  <cs.jetty.version>9.4.51.v20230217</cs.jetty.version>
  <cs.jetty-maven-plugin.version>9.4.27.v20200227</cs.jetty-maven-plugin.version>
@@ -1040,6 +1042,7 @@
  <exclude>systemvm/agent/js/jquery.js</exclude>
  <exclude>systemvm/agent/js/jquery.flot.navigate.js</exclude>
  <exclude>systemvm/agent/noVNC/**</exclude>
+ <exclude>systemvm/agent/packages/**</exclude>
  <exclude>systemvm/debian/**</exclude>
  <exclude>test/integration/component/test_host_ha.sh</exclude>
  <exclude>test/systemvm/README.md</exclude>

scripts/installer/cloudstack-help-text

@@ -40,4 +40,10 @@ printf " * Release notes: https://docs.cloudstack.apache.org/en/${ACL_MINO
 printf " * Join mailing lists: https://cloudstack.apache.org/mailing-lists.html\n"
 printf " * Take the survey: https://cloudstack.apache.org/survey.html\n"
 printf " * Report issues: https://github.com/apache/cloudstack/issues/new\n"
+
+if [ "$1" = "management" ];then
+ printf "\nSince Apache CloudStack 4.20.0.0, the System VMs and virtual routers require at least 512 MiB memory, please check the System Offerings."
+ printf "\nMore information can be found at https://docs.cloudstack.apache.org/en/${ACL_MINOR_VERSION:-latest}/upgrading/upgrade/_sysvm_restart.html\n"
+fi
+
 printf "\n"

server/src/main/java/com/cloud/configuration/Config.java

@@ -959,7 +959,7 @@ public enum Config {
  ManagementServer.class,
  Integer.class,
  "network.loadbalancer.basiczone.elb.vm.ram.size",
- "128",
+ "512",
  "Memory in MB for the elastic load balancer vm",
  null),
  ElasticLoadBalancerVmCpuMhz(
@@ -1291,7 +1291,7 @@ public enum Config {
  "The allowable clock difference in milliseconds between when an SSO login request is made and when it is received.",
  null),
  //NetworkType("Hidden", ManagementServer.class, String.class, "network.type", "vlan", "The type of network that this deployment will use.", "vlan,direct"),
- RouterRamSize("Hidden", NetworkOrchestrationService.class, Integer.class, "router.ram.size", "256", "Default RAM for router VM (in MB).", null),
+ RouterRamSize("Hidden", NetworkOrchestrationService.class, Integer.class, "router.ram.size", "512", "Default RAM for router VM (in MB).", null),
 
  DefaultPageSize("Advanced", ManagementServer.class, Long.class, "default.page.size", "500", "Default page size for API list* commands", null),
 

systemvm/agent/noVNC/keymaps/generate-language-keymaps.py

@@ -95,7 +95,7 @@ def generate_js_file(keymap_file):
  js_config.append(" * layout : %s\n" % layout)
  js_config.append(" */\n")
  js_config.append("export default {\n")
- for keycode in dict(sorted(result_mappings.items(), key=lambda item: int(item[0]))):
+ for keycode in dict(sorted(list(result_mappings.items()), key=lambda item: int(item[0]))):
  js_config.append("%10s : \"%s\",\n" % ("\"" + str(keycode) + "\"", result_mappings[keycode].strip()))
  js_config.append("}\n")
  for line in js_config:

systemvm/agent/packages/packages.ini

@@ -0,0 +1,11 @@
+[python-is-python3]
+debian_os=11
+package_name=python-is-python3
+file_name=python-is-python3_3.9.2-1_all.deb
+conflicted_packages=python-is-python2
+
+[python3-netaddr]
+debian_os=11
+package_name=python3-netaddr
+file_name=python3-netaddr_0.7.19-5_all.deb
+conflicted_packages=

systemvm/debian/etc/apache2/vhost.template

@@ -93,7 +93,7 @@
 	# Enable/Disable SSL for this virtual host.
 	SSLEngine on
 	SSLProtocol TLSv1.2
-	SSLCipherSuite @SECLEVEL=1:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+	SSLCipherSuite @SECLEVEL=0:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
 	SSLHonorCipherOrder on
 
 	# A self-signed (snakeoil) certificate can be created by installing

systemvm/debian/opt/cloud/bin/baremetal-vr.py

@@ -59,8 +59,8 @@ def __call__(self, is_exception=True):
  err = []
  err.append('failed to execute shell command: %s' % self.cmd)
  err.append('return code: %s' % self.process.returncode)
- err.append('stdout: %s' % self.stdout)
- err.append('stderr: %s' % self.stderr)
+ err.append('stdout: %s' % self.stdout.decode())
+ err.append('stderr: %s' % self.stderr.decode())
  raise Exception('\n'.join(err))
 
  self.return_code = self.process.returncode