From 3969a4d72b0f31dcbb82f96891ca24217a1a9751 Mon Sep 17 00:00:00 2001 From: Aravind Patnam Date: Fri, 26 Sep 2025 11:59:21 -0700 Subject: [PATCH 1/4] [CELEBORN-2162] Supress client side connection error logging --- .../celeborn/common/client/MasterClient.java | 2 +- .../client/TransportClientFactory.java | 39 +++++++++++++------ .../RegistrationClientBootstrap.java | 2 +- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/org/apache/celeborn/common/client/MasterClient.java b/common/src/main/java/org/apache/celeborn/common/client/MasterClient.java index 05897aab764..626c112a506 100644 --- a/common/src/main/java/org/apache/celeborn/common/client/MasterClient.java +++ b/common/src/main/java/org/apache/celeborn/common/client/MasterClient.java @@ -272,7 +272,7 @@ private RpcEndpointRef setupEndpointRef(String endpoint) { // Catch all exceptions. Because we don't care whether this exception is IOException or // TimeoutException or other exceptions, so we just try to connect to host:port, if fail, // we try next address. - LOG.warn("Connect to {} failed.", endpoint, e); + LOG.info("Connect to {} failed.", endpoint); } return endpointRef; } diff --git a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java index f9bd7335548..f3a9f25f1ac 100644 --- a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java +++ b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java @@ -38,6 +38,8 @@ import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; +import org.apache.celeborn.common.client.MasterNotLeaderException; +import org.apache.celeborn.common.network.sasl.registration.RegistrationClientBootstrap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -350,19 +352,34 @@ public void operationComplete(final Future handshakeFuture) { // Execute any client bootstraps synchronously before marking the Client as successful. long preBootstrap = System.nanoTime(); logger.debug("Running bootstraps for {} ...", address); - try { - for (TransportClientBootstrap clientBootstrap : clientBootstraps) { + for (TransportClientBootstrap clientBootstrap : clientBootstraps) { + try { clientBootstrap.doBootstrap(client); + } catch ( + Exception e) { // catch non-RuntimeExceptions too as bootstrap may be written in Scala + long bootstrapTime = System.nanoTime() - preBootstrap; + if (clientBootstrap instanceof RegistrationClientBootstrap) { + Exception processed = RegistrationClientBootstrap.processMasterNotLeaderException(e); + String message = + (processed instanceof MasterNotLeaderException) + ? String.format( + "Suggested leader is %s", + ((MasterNotLeaderException) processed).getSuggestedLeaderAddress()) + : e.getMessage(); + logger.info( + "Attempted to register with a Master that is not the leader after {}: {}", + Utils.nanoDurationToString(bootstrapTime), + message); + } else { + logger.error( + "Exception while bootstrapping client after {}", + Utils.nanoDurationToString(bootstrapTime), + e); + } + client.close(); + Throwables.throwIfUnchecked(e); + throw new RuntimeException(e); } - } catch (Exception e) { // catch non-RuntimeExceptions too as bootstrap may be written in Scala - long bootstrapTime = System.nanoTime() - preBootstrap; - logger.error( - "Exception while bootstrapping client after {}", - Utils.nanoDurationToString(bootstrapTime), - e); - client.close(); - Throwables.throwIfUnchecked(e); - throw new RuntimeException(e); } long postBootstrap = System.nanoTime(); logger.debug( diff --git a/common/src/main/java/org/apache/celeborn/common/network/sasl/registration/RegistrationClientBootstrap.java b/common/src/main/java/org/apache/celeborn/common/network/sasl/registration/RegistrationClientBootstrap.java index 8d0bd7e1aa2..9a29b0fb2b7 100644 --- a/common/src/main/java/org/apache/celeborn/common/network/sasl/registration/RegistrationClientBootstrap.java +++ b/common/src/main/java/org/apache/celeborn/common/network/sasl/registration/RegistrationClientBootstrap.java @@ -276,7 +276,7 @@ private static Map> findSupportedSaslMechs( Pattern.MULTILINE | Pattern.DOTALL); @SuppressWarnings("NonInclusiveLanguage") - static Exception processMasterNotLeaderException(Exception ex) { + public static Exception processMasterNotLeaderException(Exception ex) { String stringified = Throwables.getStackTraceAsString(ex); Matcher matcher = MASTER_NOT_LEADER_EXCEPTION_PATTERN.matcher(stringified); From 25b632677b9263d05c54a014015c8db9a1801b17 Mon Sep 17 00:00:00 2001 From: Aravind Patnam Date: Fri, 26 Sep 2025 12:10:28 -0700 Subject: [PATCH 2/4] Spotless --- .../common/network/client/TransportClientFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java index f3a9f25f1ac..18205e633ce 100644 --- a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java +++ b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java @@ -38,13 +38,13 @@ import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; -import org.apache.celeborn.common.client.MasterNotLeaderException; -import org.apache.celeborn.common.network.sasl.registration.RegistrationClientBootstrap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.celeborn.common.client.MasterNotLeaderException; import org.apache.celeborn.common.exception.CelebornIOException; import org.apache.celeborn.common.network.TransportContext; +import org.apache.celeborn.common.network.sasl.registration.RegistrationClientBootstrap; import org.apache.celeborn.common.network.server.TransportChannelHandler; import org.apache.celeborn.common.network.util.*; import org.apache.celeborn.common.util.JavaUtils; @@ -363,8 +363,8 @@ public void operationComplete(final Future handshakeFuture) { String message = (processed instanceof MasterNotLeaderException) ? String.format( - "Suggested leader is %s", - ((MasterNotLeaderException) processed).getSuggestedLeaderAddress()) + "Suggested leader is %s", + ((MasterNotLeaderException) processed).getSuggestedLeaderAddress()) : e.getMessage(); logger.info( "Attempted to register with a Master that is not the leader after {}: {}", From d0534e6974d6f758b94a97daffd7cef9ecc37a83 Mon Sep 17 00:00:00 2001 From: Aravind Patnam Date: Wed, 1 Oct 2025 15:06:39 -0700 Subject: [PATCH 3/4] convert to warn --- .../celeborn/common/network/client/TransportClientFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java index 18205e633ce..370b3e62aee 100644 --- a/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java +++ b/common/src/main/java/org/apache/celeborn/common/network/client/TransportClientFactory.java @@ -366,7 +366,7 @@ public void operationComplete(final Future handshakeFuture) { "Suggested leader is %s", ((MasterNotLeaderException) processed).getSuggestedLeaderAddress()) : e.getMessage(); - logger.info( + logger.warn( "Attempted to register with a Master that is not the leader after {}: {}", Utils.nanoDurationToString(bootstrapTime), message); From 842e3f51e203bf7f707992e500528280bdf8315c Mon Sep 17 00:00:00 2001 From: Aravind Patnam Date: Wed, 1 Oct 2025 19:38:21 -0700 Subject: [PATCH 4/4] trigger tests