<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<base href="https://issues.shibboleth.net/jira">
<title>Message Title</title>
</head>
<body class="jira" style="color: #333333; font-family: Arial, sans-serif; font-size: 14px; line-height: 1.429">
<table id="background-table" cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; background-color: #f5f5f5; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt" bgcolor="#f5f5f5">
<!-- header here -->
<tbody>
<tr>
<td id="header-pattern-container" style="padding: 0px; border-collapse: collapse; padding: 10px 20px">
<table id="header-pattern" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<td id="header-avatar-image-container" valign="top" style="padding: 0px; border-collapse: collapse; vertical-align: top; width: 32px; padding-right: 8px" width="32"> <img id="header-avatar-image" class="image_fix" src="cid:jira-generated-image-avatar-dd3b4435-4d09-40d4-9da8-97de5969a957" height="32" width="32" border="0" style="border-radius: 3px; vertical-align: top"> </td>
<td id="header-text-container" valign="middle" style="padding: 0px; border-collapse: collapse; vertical-align: middle; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px"> <a class="user-hover" rel="henson@cpp.edu" id="email_henson@cpp.edu" href="https://issues.shibboleth.net/jira/secure/ViewProfile.jspa?name=henson%40cpp.edu" style="color:#0052cc;; color: #3b73af; text-decoration: none">Paul B. Henson</a> <strong>created</strong> an issue </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="email-content-container" style="padding: 0px; border-collapse: collapse; padding: 0 20px">
<table id="email-content-table" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; border-spacing: 0; border-collapse: separate">
<tbody>
<tr>
<!-- there needs to be content in the cell for it to render in some clients -->
<td class="email-content-rounded-top mobile-expand" style="padding: 0px; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 15px; background-color: #ffffff; border-left: 1px solid #cccccc; border-top: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom: 0; border-top-right-radius: 5px; border-top-left-radius: 5px; height: 10px; line-height: 10px; padding: 0 15px 0 16px; mso-line-height-rule: exactly" height="10" bgcolor="#ffffff"> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand " style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff">
<table class="page-title-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<td class="page-title-pattern-first-line " style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; padding-top: 10px"> <a href="https://issues.shibboleth.net/jira/browse/IDP" style="color: #3b73af; text-decoration: none">Identity Provider</a> / <a href="https://issues.shibboleth.net/jira/browse/IDP-1633" style="color: #3b73af; text-decoration: none"><img src="cid:jira-generated-image-avatar-5d572ed7-ca65-4973-8b9c-0c38f6f26829" height="16" width="16" border="0" align="absmiddle" alt="Bug" style="vertical-align: text-bottom"></a> <a href="https://issues.shibboleth.net/jira/browse/IDP-1633" style="color: #3b73af; text-decoration: none">IDP-1633</a> </td>
</tr>
<tr>
<td style="vertical-align: top;; padding: 0px; border-collapse: collapse; padding-right: 5px; font-size: 20px; line-height: 30px; mso-line-height-rule: exactly" class="page-title-pattern-header-container"> <span class="page-title-pattern-header" style="font-family: Arial, sans-serif; padding: 0; font-size: 20px; line-height: 30px; mso-text-raise: 2px; mso-line-height-rule: exactly; vertical-align: middle"> <a href="https://issues.shibboleth.net/jira/browse/IDP-1633" style="color: #3b73af; text-decoration: none">httpclient sockets stuck in CLOSE_WAIT break CAS proxy callback</a> </span> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand wrapper-special-margin" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff; padding-top: 10px; padding-bottom: 5px" bgcolor="#ffffff">
<table class="keyvalue-table" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Issue Type:</th>
<td class="has-icon" style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <img src="cid:jira-generated-image-avatar-5d572ed7-ca65-4973-8b9c-0c38f6f26829" height="16" width="16" border="0" align="absmiddle" alt="Bug" style="vertical-align: text-bottom"> Bug </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Assignee:</th>
<td style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <a class="user-hover" rel="serac@vt.edu" id="email_serac@vt.edu" href="https://issues.shibboleth.net/jira/secure/ViewProfile.jspa?name=serac%40vt.edu" style="color:#0052cc;; color: #3b73af; text-decoration: none">Marvin S Addison</a> </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Components:</th>
<td style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> CAS </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Created:</th>
<td style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> 01/Jul/20 9:20 PM </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Priority:</th>
<td class="has-icon" style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <img src="cid:jira-generated-image-static-major-e94378b0-4363-4983-82e6-3c9eff0dd0f6" height="16" width="16" border="0" align="absmiddle" alt="Major" style="vertical-align: text-bottom"> Major </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Reporter:</th>
<td style="padding: 0px; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <a class="user-hover" rel="henson@cpp.edu" id="email_henson@cpp.edu" href="https://issues.shibboleth.net/jira/secure/ViewProfile.jspa?name=henson%40cpp.edu" style="color:#0052cc;; color: #3b73af; text-decoration: none">Paul B. Henson</a> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand issue-description-container" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff; padding-top: 5px; padding-bottom: 10px" bgcolor="#ffffff">
<table class="text-paragraph-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 2px">
<tbody>
<tr>
<td class="text-paragraph-pattern-container mobile-resize-text " style="padding: 0px; border-collapse: collapse; padding: 0 0 10px 0"> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0; margin-top: 0">After recently installing some updates on our infrastructure, I ran into a problem similar to issue <a href="https://issues.shibboleth.net/jira/browse/IDP-1483" title="CAS Proxy authentication failures require IDP restart to resolve." class="issue-link" data-issue-key="IDP-1483" style="color: #3b73af; text-decoration: none"><del>IDP-1483</del></a>, where sockets from the httpclient callback to the CAS proxy client end up in CLOSE_WAIT state. However, unlike that issue, where the reporter says they time out after about five minutes and go away, in my case, they appear to be lasting forever 8-/. Once the pool is full of stale sockets, callbacks start to fail:</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">2020-06-29 13:30:34,779 - 2620:df:8000:ff10:0:1:247:16/86C949E906E495B3A7742481EB67A7E9 - INFO <span class="error">[net.shibboleth.idp.cas.flow.impl.ValidateProxyCallbackAction:139]</span> - Proxy authentication failed for <a href="https://www.idm.unx.cpp.edu/cas_pgt:" class="external-link" target="_blank" rel="nofollow noopener" style="color: #3b73af; text-decoration: none">https://www.idm.unx.cpp.edu/cas_pgt:</a> java.security.GeneralSecurityException: IO error</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">and the instance must be restarted to recover. The previous issue was closed indicating it was an environmental issue with nothing to fix. However, while it clearly is triggered by some specific circumstances, it does seem like a bug on the IDP side, though exactly where I can't say.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">Basically, the web server on the CAS client side closes the connection with a FIN, but the http client on the idp never closes its side, leaving the connection in half open state. A packet capture shows the connection starts off normally enough with a <br> SYN from the idp to the client, a SYN-ACK back, and a final ACK. Then TLS is negotiated and the content transfer completed. The client sends a packet with both ACK (acknowledging the last packet from the idp) and FIN (closing the connection) set. The idp sends an ACK for the FIN packet, but the idp never sends a FIN of its own, leaving the socket in CLOSE_WAIT state.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">A socket stuck in CLOSE_WAIT state would appear to be an application issue:</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0"><a href="https://doc.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connect" class="external-link" target="_blank" rel="nofollow noopener" style="color: #3b73af; text-decoration: none">https://doc.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connect</a><br> ions/</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0"><a href="https://stackoverflow.com/questions/5636316/troubleshooting-connections-stuck-in" class="external-link" target="_blank" rel="nofollow noopener" style="color: #3b73af; text-decoration: none">https://stackoverflow.com/questions/5636316/troubleshooting-connections-stuck-in</a><br> -close-wait-status</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">The socket is not being closed. This bug looks similar: </p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0"><a href="https://issues.apache.org/jira/browse/HTTPCLIENT-1887" class="external-link" target="_blank" rel="nofollow noopener" style="color: #3b73af; text-decoration: none">https://issues.apache.org/jira/browse/HTTPCLIENT-1887</a></p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">It says the pool eviction thread should call Socket.close()? Where in the idp is this? Or does the idp rely on the default ttpClient check for staleness executed when a request is made? I see the use of an httpclient in idp/cas/proxy/impl/HttpClientProxyValidator.java but my java/spring fu isn't quite up to tracing where the passed in object comes from <sigh>.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">I'm using a custom bean now for CAS proxy validation as a workaround: </p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0"><bean id="shibboleth.CASProxyValidatorHttpClient" <br> parent="shibboleth.NonCachingHttpClient" <br> p:tLSSocketFactory-ref="shibboleth.SecurityEnhancedTLSSocketFactory" <br> p:maxConnectionsTotal="500" <br> p:maxConnectionsPerRoute="500" /> </p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">Is there some setting I could use here to make the connections close/get evicted properly? </p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0 0">This situation only occurs when the remote CAS client web server closes the connection, not when the idp closes the connection. What might cause that? Is that the expected behavior or an unusual behavior? Perhaps that's the environmental difference that causes this problem.</p> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand " style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff">
<table id="actions-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px">
<tbody>
<tr>
<td id="actions-pattern-container" valign="middle" style="padding: 0px; border-collapse: collapse; padding: 10px 0 10px 24px; vertical-align: middle; padding-left: 0">
<table align="left" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<td class="actions-pattern-action-icon-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0; vertical-align: middle"> <a href="https://issues.shibboleth.net/jira/browse/IDP-1633#add-comment" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none"> <img class="actions-pattern-action-icon-image" src="cid:jira-generated-image-static-comment-icon-6f5fc903-0050-4223-a608-73c1c3832760" alt="Add Comment" title="Add Comment" height="16" width="16" border="0" style="vertical-align: middle"> </a> </td>
<td class="actions-pattern-action-text-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px"> <a href="https://issues.shibboleth.net/jira/browse/IDP-1633#add-comment" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none">Add Comment</a> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table> </td>
</tr>
<!-- there needs to be content in the cell for it to render in some clients -->
<tr>
<td class="email-content-rounded-bottom mobile-expand" style="padding: 0px; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 5px; line-height: 5px; background-color: #ffffff; border-top: 0; border-left: 1px solid #cccccc; border-bottom: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom-right-radius: 5px; border-bottom-left-radius: 5px; mso-line-height-rule: exactly" height="5" bgcolor="#ffffff"> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="footer-pattern" style="padding: 0px; border-collapse: collapse; padding: 12px 20px">
<table id="footer-pattern-container" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<td id="footer-pattern-text" class="mobile-resize-text" width="100%" style="padding: 0px; border-collapse: collapse; color: #999999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px"> This message was sent by Atlassian Jira <span id="footer-build-information">(v8.5.4#805004-<span title="0444eab799707f9ad7b248d69f858774aadfd250" data-commit-id="0444eab799707f9ad7b248d69f858774aadfd250}">sha1:0444eab</span>)</span> </td>
<td id="footer-pattern-logo-desktop-container" valign="top" style="padding: 0px; border-collapse: collapse; padding-left: 20px; vertical-align: top">
<table style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt">
<tbody>
<tr>
<td id="footer-pattern-logo-desktop-padding" style="padding: 0px; border-collapse: collapse; padding-top: 3px"> <img id="footer-pattern-logo-desktop" src="https://issues.shibboleth.net/jira/images/mail/atlassian-email-logo.png" alt="Atlassian logo" title="Atlassian logo" width="191" height="24" class="image_fix"> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table>
</body>
</html>