<!-- BaNnErBlUrFlE-BoDy-start -->
<!-- Preheader Text : BEGIN -->
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">
Hi All, A user on our HPC cluster is reporting this issue - Has anyone come across this, or has any ideas to solve this? Note: The workflow is executing without issues using OpenMP, although slower. Workflow: User is launching mpirun jobs in
</div>
<!-- Preheader Text : END -->
<!-- Email Banner : BEGIN -->
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">ZjQcmQRYFpfptBannerStart</div>
<!--[if ((ie)|(mso))]>
<table border="0" cellspacing="0" cellpadding="0" width="100%" style="padding: 16px 0px 16px 0px; direction: ltr" lang="en"><tr><td>
<table border="0" cellspacing="0" cellpadding="0" style="padding: 0px 10px 5px 6px; width: 100%; border-radius:4px; border-top:4px solid #8c8e91;background-color:#CFD3D7;"><tr><td valign="top">
<table align="left" border="0" cellspacing="0" cellpadding="0" style="padding: 4px 8px 4px 8px">
<tr><td style="color:#000000; font-family: 'Arial', sans-serif; font-weight:bold; font-size:14px; direction: ltr">
This Message Is From an External Sender
</td></tr>
<tr><td style="color:#000000; font-weight:normal; font-family: 'Arial', sans-serif; font-size:12px; direction: ltr">
This message came from outside your organization.
</td></tr>
</table>
<![if ie]><br clear="all"><![endif]>
<table align="right" border="0" cellspacing="0" cellpadding="0" style="padding: 4px 0px 4px 0px"><tr>
<td style="direction: ltr"> <a target="_blank" href="https://us-phishalarm-ewt.proofpoint.com/EWT/v1/KGKeukY!vwQd8katD6YAhRdxny5E-Zaa_y0GgRJWJx4oj4IcWekuVBBq2b4PW5XcZzyjIEnjzRwRsUevJovLgqsOy6npjn3zHwMOheLKfs5zEGIsogW40MqmdiR0fdfcEQzFcyLKb2kRZcvPyJ2Mcm22LsIftg$" style="mso-padding-alt: 7.5px; padding: 7.5px; border-radius: 2px; border: 1.5px solid #666666; "><strong style="font-weight: normal; color: #000000; text-decoration: none; font-family: 'Arial', sans-serif; font-size:14px; line-height: 40px; "> Report Suspicious </strong></a> </td>
</tr></table>
</td></tr></table>
</td></tr></table>
<![endif]-->
<![if !((ie)|(mso))]>
<div dir="ltr" lang="en" id="pfptBannerxwkvvfb" style="all: revert !important; display:block !important; text-align: left !important; margin:16px 0px 16px 0px !important; padding:8px 16px 8px 16px !important; border-radius: 4px !important; min-width: 200px !important; background-color: #CFD3D7 !important; background-color: #CFD3D7; border-top: 4px solid #8c8e91 !important; border-top: 4px solid #8c8e91;">
<div id="pfptBannerxwkvvfb" style="all: unset !important; float:left !important; display:block !important; margin: 0px 0px 1px 0px !important; max-width: 600px !important;">
<div id="pfptBannerxwkvvfb" style="all: unset !important; display:block !important; visibility: visible !important; background-color: #CFD3D7 !important; color:#000000 !important; color:#000000; font-family: 'Arial', sans-serif !important; font-family: 'Arial', sans-serif; font-weight:bold !important; font-weight:bold; font-size:14px !important; line-height:18px !important; line-height:18px">
This Message Is From an External Sender
</div>
<div id="pfptBannerxwkvvfb" style="all: unset !important; display:block !important; visibility: visible !important; background-color: #CFD3D7 !important; color:#000000 !important; color:#000000; font-weight:normal; font-family: 'Arial', sans-serif !important; font-family: 'Arial', sans-serif; font-size:12px !important; line-height:18px !important; line-height:18px; margin-top:2px !important;">
This message came from outside your organization.
</div>
</div>
<div id="pfptBannerxwkvvfb" style="all: unset !important; float: right !important; display: block !important; display: block; margin: 0px 0px 0px 16px !important; text-align: right !important; width: fit-content !important;">
<a id="pfptBannerxwkvvfb" href="https://us-phishalarm-ewt.proofpoint.com/EWT/v1/KGKeukY!vwQd8katD6YAhRdxny5E-Zaa_y0GgRJWJx4oj4IcWekuVBBq2b4PW5XcZzyjIEnjzRwRsUevJovLgqsOy6npjn3zHwMOheLKfs5zEGIsogW40MqmdiR0fdfcEQzFcyLKb2kRZcvPyJ2Mcm22LsIftg$"
style="all: unset !important; display: inline-block !important; text-decoration: none">
<div class="pfptPrimaryButtonxwkvvfb" style="display: inline-block !important; display: inline-block; visibility: visible !important; opacity: 1 !important; color: #000000 !important; color: #000000; font-family: 'Arial', sans-serif !important; font-family: 'Arial', sans-serif; font-size: 14px !important; font-weight: normal !important; text-decoration: none !important; border-radius: 2px !important; padding: 7.5px 16px !important; margin: 3px 0 3px 16px !important; white-space: nowrap !important; width: fit-content !important;
border: 1px solid #666666">
Report Suspicious
</div>
</a>
</div>
<div style="clear: both !important; display: block !important; visibility: hidden !important; line-height: 0 !important; font-size: 0.01px !important; height: 0px"> </div>
</div>
<![endif]>
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">ZjQcmQRYFpfptBannerEnd</div>
<!-- Email Banner : END -->
<!-- BaNnErBlUrFlE-BoDy-end -->
<html>
<head><!-- BaNnErBlUrFlE-HeAdEr-start -->
<style>
#pfptBannerxwkvvfb { all: revert !important; display: block !important;
visibility: visible !important; opacity: 1 !important;
background-color: #CFD3D7 !important;
max-width: none !important; max-height: none !important }
.pfptPrimaryButtonxwkvvfb:hover, .pfptPrimaryButtonxwkvvfb:focus {
background-color: #adb0b4 !important; }
.pfptPrimaryButtonxwkvvfb:active {
background-color: #8c8e91 !important; }
</style>
<!-- BaNnErBlUrFlE-HeAdEr-end -->
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>
</head>
<body dir="ltr">
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Hi All,</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
A user on our HPC cluster is reporting this issue - Has anyone come across this, or has any ideas to solve this?</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<i>Note: The workflow is executing without issues using OpenMP, although slower.</i></div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Workflow: User is launching mpirun jobs in series within a loop (1 PBS job per time step). Each job reads from input file, computes and writes to output, which is read by the next job.</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Issue: At sporadic times, he is getting the error message "HYDT_bscd_pbs_wait_for_completion (tools/bootstrap/external/pbs_wait.c:67): tm_poll(obit_event) failed with TM error 17002" where the program is failing to write the output file on a random time step. </div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Debugging efforts so far: 1) We have tried adding sleep intervals between job launch, that didn't help. 2) We have added barriers before MPI_Finalize, that didn't help. 3) We have checked the stack size, that looks good. 4) We have run simulations without file
write, and just reads. That still has issues. I have attached the debug messages below showing where it fails.</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Any help is much appreciated!</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">For reference, the output was obtained with the following
run options in the pbs script:</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">mpirun -np 4 -genv I_MPI_HYDRA_DEBUG=1 -verbose $WKDIR/circ3d.x</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">Here's the tail end of the output from timestep 50, which
is the last one that completed successfully before the thing failed at the end of the next time step:</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 4): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 5): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 6): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 9): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] forwarding
command (cmd=barrier_in) upstream</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] [pgid: 0]
got PMI command: cmd=barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] PMI response
to fd 7 pid 9: cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 4): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 5): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 6): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 9): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu]</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">PBS_DEBUG: Done with polling obit events</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important"> Job ended at Tue Aug 2 11:02:51 EDT 2022</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">And here is the same block of output from the next time
step, where it failed:</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 4): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 5): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 6): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 9): barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] forwarding
command (cmd=barrier_in) upstream</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] [pgid: 0]
got PMI command: cmd=barrier_in</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] PMI response
to fd 7 pid 9: cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=barrier_out</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 9): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 5): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 6): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] got pmi
command (from 4): finalize</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[proxy:0:0@atl1-1-02-010-15-l.pace.gatech.edu] PMI response:
cmd=finalize_ack</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] HYDT_bscd_pbs_wait_for_completion
(tools/bootstrap/external/pbs_wait.c:67): tm_poll(obit_event) failed with TM error 17002</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] HYDT_bsci_wait_for_completion
(tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] HYD_pmci_wait_for_completion
(pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<span style="color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255);display:inline !important">[mpiexec@atl1-1-02-010-15-l.pace.gatech.edu] main (ui/mpich/mpiexec.c:340):
process manager error waiting for completion</span><br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
<br style="box-sizing:border-box;color:rgb(69, 84, 100);font-family:SourceSansPro, "Helvetica Neue", Helvetica, Arial, sans-serif;font-size:12px;text-align:left;background-color:rgb(255, 255, 255)">
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Regards,</div>
<div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div id="Signature">
<div>
<div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<p style="font-size:12pt;text-align:start;color:rgb(32, 31, 30);font-family:Calibri, sans-serif;background-color:white;margin:0px">
<span style="margin:0px;color:black">Deepa Phanish, PhD</span></p>
<p style="font-size:12pt;text-align:start;color:rgb(32, 31, 30);font-family:Calibri, sans-serif;background-color:white;margin:0px">
<span style="font-size:10pt;margin:0px;color:black">Research Scientist II, Research Computing Facilitator</span></p>
<p style="font-size:12pt;text-align:start;color:rgb(32, 31, 30);font-family:Calibri, sans-serif;background-color:white;margin:0px">
<span style="font-size:10pt;margin:0px;color:black">Partnership for an Advanced Computing Environment (PACE)</span></p>
<p style="font-size:12pt;text-align:start;color:rgb(32, 31, 30);font-family:Calibri, sans-serif;background-color:white;margin:0px">
<span style="font-size:10pt;margin:0px;color:black">Georgia Institute of Technology</span></p>
<br>
</div>
</div>
</div>
</div>
</body>
</html>