[ConsoleProxy] Performance fixes on concurrency and closing console sessions sockets and introduce reconnection window for console sessions

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java

@@ -84,6 +84,9 @@ public interface ConsoleProxyManager extends Manager, ConsoleProxyService {
  ConfigKey<Boolean> ConsoleProxyDisableRpFilter = new ConfigKey<>(Boolean.class, "consoleproxy.disable.rpfilter", "Console Proxy", "true",
  "disable rp_filter on console proxy VM public interface", true, ConfigKey.Scope.Zone, null);
 
+ ConfigKey<Long> ConsoleProxySessionReconnectionWindow = new ConfigKey<>(Long.class, "consoleproxy.session.reconnection.window", "Console Proxy", "0",
+ "Reconnection window (in milliseconds) for client IPs to the same session on console proxy VM", true, ConfigKey.Scope.Zone, null);
+
  ConfigKey<Integer> ConsoleProxyLaunchMax = new ConfigKey<>(Integer.class, "consoleproxy.launch.max", "Console Proxy", "10",
  "maximum number of console proxy instances per zone can be launched", false, ConfigKey.Scope.Zone, null);
 

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -1210,6 +1210,10 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
  if (Boolean.TRUE.equals(disableRpFilter)) {
  buf.append(" disable_rp_filter=true");
  }
+ Long sessionReconnectionWindow = ConsoleProxySessionReconnectionWindow.valueIn(datacenterId);
+ if (sessionReconnectionWindow != null && sessionReconnectionWindow > 0) {
+ buf.append(" session_reconnection_window=").append(sessionReconnectionWindow);
+ }
 
  String msPublicKey = configurationDao.getValue("ssh.publickey");
  buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey));
@@ -1588,7 +1592,7 @@ public ConfigKey<?>[] getConfigKeys() {
  return new ConfigKey<?>[] {ConsoleProxySslEnabled, NoVncConsoleDefault, NoVncConsoleSourceIpCheckEnabled, ConsoleProxyServiceOffering,
  ConsoleProxyCapacityStandby, ConsoleProxyCapacityScanInterval, ConsoleProxyRestart, ConsoleProxyUrlDomain, ConsoleProxySessionMax, ConsoleProxySessionTimeout, ConsoleProxyDisableRpFilter, ConsoleProxyLaunchMax,
  ConsoleProxyManagementLastState, ConsoleProxyServiceManagementState, NoVncConsoleShowDot,
- ConsoleProxyVmUserData};
+ ConsoleProxyVmUserData, ConsoleProxySessionReconnectionWindow};
  }
 
  protected ConsoleProxyStatus parseJsonToConsoleProxyStatus(String json) throws JsonParseException {

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java

@@ -26,7 +26,6 @@
 import java.net.InetSocketAddress;
 import java.net.URISyntaxException;
 import java.net.URL;
-import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.Map;
 import java.util.Properties;
@@ -83,12 +82,73 @@ public class ConsoleProxy {
  static String encryptorPassword = "Dummy";
  static final String[] skipProperties = new String[]{"certificate", "cacertificate", "keystore_password", "privatekey"};
 
- static Set<String> allowedSessions = new HashSet<>();
+ static Set<String> allowedSessions = ConcurrentHashMap.newKeySet();
+ private static final Object allowedSessionsLock = new Object();
 
+ private static final Map<String, ReconnectGrant> sessionReconnectGrants = new ConcurrentHashMap<>();
+ private static long sessionReconnectionWindowMs = 0L;
+
+ // Invoked through reflection
  public static void addAllowedSession(String sessionUuid) {
  allowedSessions.add(sessionUuid);
  }
 
+ /**
+ * Grant the client IP a reconnection window of #{@link #sessionReconnectionWindowMs} ms to the same session UUID in case of a disconnection.
+ * The grant is bound to the client IP that was using the session so it cannot be redeemed by another client.
+ * @param sessionUuid session UUID to grant a reconnect window for
+ * @param clientIp source IP of the client the session was granted to
+ */
+ public static void grantReconnectWindowForSessionAndClientIp(String sessionUuid, String clientIp) {
+ if (sessionReconnectionWindowMs > 0) {
+ ReconnectGrant grant = new ReconnectGrant(System.currentTimeMillis() + sessionReconnectionWindowMs, clientIp);
+ sessionReconnectGrants.put(sessionUuid, grant);
+ }
+ }
+
+ /**
+ * True if the session UUID has been granted reconnection, within the reconnection window #{@link #sessionReconnectionWindowMs}.
+ */
+ private static boolean isSessionReconnectionGrantedForClientIp(String sessionUuid, String clientIp) {
+ ReconnectGrant grant = sessionReconnectGrants.remove(sessionUuid);
+ if (grant == null) {
+ return false;
+ }
+ if (grant.isExpired(System.currentTimeMillis())) {
+ LOGGER.warn("Rejecting reconnection for session {} as the reconnect window: {}ms is already expired",
+ sessionUuid, sessionReconnectionWindowMs);
+ return false;
+ }
+ if (grant.clientIp != null && !grant.clientIp.equals(clientIp)) {
+ LOGGER.warn("Rejecting reconnection for session {} as it was requested from IP {} " +
+ "but the session was granted to IP {}", sessionUuid, clientIp, grant.clientIp);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Drops expired, unclaimed reconnect grants so sessionReconnectGrants doesn't grow unbounded
+ * when a client never reconnects after a disconnection. Invoked periodically by {@link ConsoleProxyGCThread}.
+ */
+ static void cleanupExpiredReconnectGrants() {
+ sessionReconnectGrants.entrySet().removeIf(entry -> entry.getValue().isExpired(System.currentTimeMillis()));
+ }
+
+ private static final class ReconnectGrant {
+ final long expiryMillis;
+ final String clientIp;
+
+ ReconnectGrant(long expiryMillis, String clientIp) {
+ this.expiryMillis = expiryMillis;
+ this.clientIp = clientIp;
+ }
+
+ boolean isExpired(long now) {
+ return now >= expiryMillis;
+ }
+ }
+
  private static void configLog4j() {
  final ClassLoader loader = Thread.currentThread().getContextClassLoader();
  URL configUrl = loader.getResource("/conf/log4j-cloud.xml");
@@ -166,6 +226,12 @@ private static void configProxy(Properties conf) {
  defaultBufferSize = Integer.parseInt(s);
  LOGGER.info("Setting defaultBufferSize=" + defaultBufferSize);
  }
+
+ s = conf.getProperty("session_reconnection_window");
+ if (s != null) {
+ sessionReconnectionWindowMs = Long.parseLong(s);
+ LOGGER.info("Setting sessionReconnectionWindowMs=" + sessionReconnectionWindowMs);
+ }
  }
 
  public static ConsoleProxyServerFactory getHttpServerFactory() {
@@ -209,13 +275,17 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console
  }
 
  String sessionUuid = param.getSessionUuid();
- if (allowedSessions.contains(sessionUuid)) {
- LOGGER.debug("Acquiring the session " + sessionUuid + " not available for future use");
- allowedSessions.remove(sessionUuid);
- } else {
- LOGGER.info("Session " + sessionUuid + " has already been used, cannot connect");
- authResult.setSuccess(false);
- return authResult;
+ synchronized (allowedSessionsLock) {
+ if (allowedSessions.remove(sessionUuid)) {
+ LOGGER.debug("Acquiring the session {} from client IP {}", sessionUuid, param.getClientIp());
+ } else if (isSessionReconnectionGrantedForClientIp(sessionUuid, param.getClientIp())) {
+ LOGGER.info("Reconnecting the session {} after a dropped connection", sessionUuid);
+ return authResult;
+ } else {
+ LOGGER.info("Invalid or already used session {}, cannot connect", sessionUuid);
+ authResult.setSuccess(false);
+ return authResult;
+ }
  }
 
  String websocketUrl = param.getWebsocketUrl();
@@ -625,7 +695,7 @@ public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam par
  } catch (IOException e) {
  LOGGER.error("Exception while disconnect session of novnc viewer object: " + viewer, e);
  }
-removeViewer(viewer);
+ viewer.closeClient();
  viewer = new ConsoleProxyNoVncClient(session);
  viewer.initClient(param);
  connectionMap.put(clientKey, viewer);

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyGCThread.java

@@ -75,6 +75,7 @@ public void run() {
 
  while (true) {
  cleanupLogging();
+ ConsoleProxy.cleanupExpiredReconnectGrants();
  bReportLoad = false;
 
  if (logger.isDebugEnabled()) {

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVNCHandler.java

@@ -175,12 +175,20 @@ private boolean checkSessionSourceIp(final Session session, final String sourceI
 
  @OnWebSocketClose
  public void onClose(Session session, int statusCode, String reason) throws IOException, InterruptedException {
- String sessionSourceIp = session.getRemoteAddress().getAddress().getHostAddress();
- logger.debug("Closing WebSocket session [source IP: {}, status code: {}].", sessionSourceIp, statusCode);
  if (viewer != null) {
- ConsoleProxy.removeViewer(viewer);
+ viewer.closeClient();
+ }
+ String sessionSourceIp = getRemoteAddressSafely(session);
+ logger.debug("WebSocket session [source IP: {}, status code: {}, reason: {}] closed successfully.", sessionSourceIp, statusCode, reason);
+ }
+
+ private String getRemoteAddressSafely(Session session) {
+ try {
+ return session.getRemoteAddress().getAddress().getHostAddress();
+ } catch (Exception e) {
+ logger.debug("Failed to get remote address from WebSocket session", e);
+ return "unknown";
  }
- logger.debug("WebSocket session [source IP: {}, status code: {}] closed successfully.", sessionSourceIp, statusCode);
  }
 
  @OnWebSocketFrame

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java

@@ -177,6 +177,7 @@ public void run() {
  }
  }
  logger.info("Connection with client [{}] [IP: {}] is dead.", clientId, clientSourceIp);
+ ConsoleProxy.grantReconnectWindowForSessionAndClientIp(sessionUuid, clientSourceIp);
  } catch (IOException e) {
  logger.error("Error on VNC client", e);
  }
@@ -374,6 +375,9 @@ public void closeClient() {
  this.connectionAlive = false;
  // Clear buffer reference to allow GC when client disconnects
  this.readBuffer = null;
+ if (client != null) {
+ client.close();
+ }
  ConsoleProxy.removeViewer(this);
  }
 

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/NoVncClient.java

@@ -129,6 +129,28 @@ public void proxyMsgOverWebSocketConnection(ByteBuffer msg) {
  }
  }
 
+ public void close() {
+ if (nioSocketConnection != null) {
+ nioSocketConnection.close();
+ }
+ if (webSocketReverseProxy != null) {
+ webSocketReverseProxy.close();
+ }
+ if (socket != null) {
+ try {
+ if (is != null) {
+ is.close();
+ }
+ if (os != null) {
+ os.close();
+ }
+ socket.close();
+ } catch (IOException e) {
+ logger.debug("Error closing socket: " + e.getMessage(), e);
+ }
+ }
+ }
+
  private void setTunnelSocketStreams() throws IOException {
  this.is = new DataInputStream(this.socket.getInputStream());
  this.os = new DataOutputStream(this.socket.getOutputStream());

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocket.java

@@ -120,4 +120,28 @@ protected int writeToSocketChannel(ByteBuffer buf, int len) {
  return 0;
  }
  }
+
+ public void close() {
+ try {
+ if (socketChannel != null) {
+ socketChannel.close();
+ }
+ } catch (IOException e) {
+ logger.debug("Error closing socket channel: " + e.getMessage(), e);
+ }
+ try {
+ if (readSelector != null) {
+ readSelector.close();
+ }
+ } catch (IOException e) {
+ logger.debug("Error closing read selector: " + e.getMessage(), e);
+ }
+ try {
+ if (writeSelector != null) {
+ writeSelector.close();
+ }
+ } catch (IOException e) {
+ logger.debug("Error closing write selector: " + e.getMessage(), e);
+ }
+ }
 }

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandler.java

@@ -41,4 +41,5 @@ public interface NioSocketHandler {
  void flushWriteBuffer();
  void startTLSConnection(NioSocketSSLEngineManager sslEngineManager);
  boolean isTLSConnection();
+ void close();
 }

services/console-proxy/server/src/main/java/com/cloud/consoleproxy/vnc/network/NioSocketHandlerImpl.java

@@ -28,10 +28,12 @@ public class NioSocketHandlerImpl implements NioSocketHandler {
  private NioSocketInputStream inputStream;
  private NioSocketOutputStream outputStream;
  private boolean isTLS = false;
+ private final NioSocket socket;
 
  protected Logger logger = LogManager.getLogger(getClass());
 
  public NioSocketHandlerImpl(NioSocket socket) {
+ this.socket = socket;
  this.inputStream = new NioSocketInputStream(ConsoleProxy.defaultBufferSize, socket);
  this.outputStream = new NioSocketOutputStream(ConsoleProxy.defaultBufferSize, socket);
  }
@@ -109,4 +111,9 @@ public NioSocketInputStream getInputStream() {
  public NioSocketOutputStream getOutputStream() {
  return outputStream;
  }
+
+ @Override
+ public void close() {
+ socket.close();
+ }
 }