<?plantuml 1.2026.2?><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" contentStyleType="text/css" data-diagram-type="SEQUENCE" height="876px" preserveAspectRatio="none" style="width:1124px;height:876px;background:#FFFFFF;" version="1.1" viewBox="0 0 1124 876" width="1124px" zoomAndPan="magnify"><defs/><g><g class="participant-lifeline" data-entity-uid="part1" data-qualified-name="user" id="part1-lifeline"><g><title>User</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="23.1826" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="27" x2="27" y1="67.4883" y2="797.873"/></g></g><g class="participant-lifeline" data-entity-uid="part2" data-qualified-name="webhook" id="part2-lifeline"><g><title>MutatingWebhook</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="229.811" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="233.3545" x2="233.3545" y1="67.4883" y2="797.873"/></g></g><g class="participant-lifeline" data-entity-uid="part3" data-qualified-name="sched" id="part3-lifeline"><g><title>Scheduler-Extender</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="429.3555" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="432.8989" x2="432.8989" y1="67.4883" y2="797.873"/></g></g><g class="participant-lifeline" data-entity-uid="part4" data-qualified-name="dp" id="part4-lifeline"><g><title>Device Plugin</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="703.231" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="706.3955" x2="706.3955" y1="67.4883" y2="797.873"/></g></g><g class="participant-lifeline" data-entity-uid="part5" data-qualified-name="core" id="part5-lifeline"><g><title>HAMi-Core</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="881.231" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="885.0664" x2="885.0664" y1="67.4883" y2="797.873"/></g></g><g class="participant-lifeline" data-entity-uid="part6" data-qualified-name="gpu" id="part6-lifeline"><g><title>Physical GPU</title><rect fill="#000000" fill-opacity="0.00000" height="730.3848" width="8" x="1062.2021" y="67.4883"/><line style="stroke:#181818;stroke-width:0.5;stroke-dasharray:5,5;" x1="1065.2725" x2="1065.2725" y1="67.4883" y2="797.873"/></g></g><g class="participant participant-head" data-entity-uid="part1" data-qualified-name="user" id="part1-head"><rect fill="#E2E2F0" height="30.4883" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="44.3652" x="5" y="36"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="30.3652" x="12" y="56.5352">User</text></g><g class="participant participant-tail" data-entity-uid="part1" data-qualified-name="user" id="part1-tail"><rect fill="#E2E2F0" height="30.4883" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="44.3652" x="5" y="796.873"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="30.3652" x="12" y="817.4082">User</text></g><g class="participant participant-head" data-entity-uid="part2" data-qualified-name="webhook" id="part2-head"><rect fill="#CCEEFF" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="166.9131" x="150.3545" y="19.5117"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="123.0742" x="172.2739" y="40.0469">MutatingWebhook</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="152.9131" x="157.3545" y="56.5352">[Cluster Control Layer]</text></g><g class="participant participant-tail" data-entity-uid="part2" data-qualified-name="webhook" id="part2-tail"><rect fill="#CCEEFF" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="166.9131" x="150.3545" y="796.873"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="123.0742" x="172.2739" y="817.4082">MutatingWebhook</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="152.9131" x="157.3545" y="833.8965">[Cluster Control Layer]</text></g><g class="participant participant-head" data-entity-uid="part3" data-qualified-name="sched" id="part3-head"><rect fill="#CCEEFF" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="166.9131" x="349.8989" y="19.5117"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="134.6133" x="366.0488" y="40.0469">Scheduler-Extender</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="152.9131" x="356.8989" y="56.5352">[Cluster Control Layer]</text></g><g class="participant participant-tail" data-entity-uid="part3" data-qualified-name="sched" id="part3-tail"><rect fill="#CCEEFF" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="166.9131" x="349.8989" y="796.873"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="134.6133" x="366.0488" y="817.4082">Scheduler-Extender</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="152.9131" x="356.8989" y="833.8965">[Cluster Control Layer]</text></g><g class="participant participant-head" data-entity-uid="part4" data-qualified-name="dp" id="part4-head"><rect fill="#CCFFCC" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="143.6709" x="635.3955" y="19.5117"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="90.918" x="661.772" y="40.0469">Device Plugin</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="129.6709" x="642.3955" y="56.5352">[Node Agent Layer]</text></g><g class="participant participant-tail" data-entity-uid="part4" data-qualified-name="dp" id="part4-tail"><rect fill="#CCFFCC" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="143.6709" x="635.3955" y="796.873"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="90.918" x="661.772" y="817.4082">Device Plugin</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="129.6709" x="642.3955" y="833.8965">[Node Agent Layer]</text></g><g class="participant participant-head" data-entity-uid="part5" data-qualified-name="core" id="part5-head"><rect fill="#FFFFCC" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="192.3291" x="789.0664" y="19.5117"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="75.9678" x="847.2471" y="40.0469">HAMi-Core</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="178.3291" x="796.0664" y="56.5352">[Container Isolation Layer]</text></g><g class="participant participant-tail" data-entity-uid="part5" data-qualified-name="core" id="part5-tail"><rect fill="#FFFFCC" height="46.9766" rx="2.5" ry="2.5" style="stroke:#181818;stroke-width:0.5;" width="192.3291" x="789.0664" y="796.873"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="75.9678" x="847.2471" y="817.4082">HAMi-Core</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="178.3291" x="796.0664" y="833.8965">[Container Isolation Layer]</text></g><g class="participant participant-head" data-entity-uid="part6" data-qualified-name="gpu" id="part6-head"><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="85.8594" x="1020.2725" y="64.5352">Physical GPU</text><path d="M1048.2021,15 C1048.2021,5 1066.2021,5 1066.2021,5 C1066.2021,5 1084.2021,5 1084.2021,15 L1084.2021,41 C1084.2021,51 1066.2021,51 1066.2021,51 C1066.2021,51 1048.2021,51 1048.2021,41 L1048.2021,15" fill="#E2E2F0" style="stroke:#181818;stroke-width:0.5;"/><path d="M1048.2021,15 C1048.2021,25 1066.2021,25 1066.2021,25 C1066.2021,25 1084.2021,25 1084.2021,15" fill="none" style="stroke:#181818;stroke-width:0.5;"/></g><g class="participant participant-tail" data-entity-uid="part6" data-qualified-name="gpu" id="part6-tail"><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="85.8594" x="1020.2725" y="810.4082">Physical GPU</text><path d="M1048.2021,823.3613 C1048.2021,813.3613 1066.2021,813.3613 1066.2021,813.3613 C1066.2021,813.3613 1084.2021,813.3613 1084.2021,823.3613 L1084.2021,849.3613 C1084.2021,859.3613 1066.2021,859.3613 1066.2021,859.3613 C1066.2021,859.3613 1048.2021,859.3613 1048.2021,849.3613 L1048.2021,823.3613" fill="#E2E2F0" style="stroke:#181818;stroke-width:0.5;"/><path d="M1048.2021,823.3613 C1048.2021,833.3613 1066.2021,833.3613 1066.2021,833.3613 C1066.2021,833.3613 1084.2021,833.3613 1084.2021,823.3613" fill="none" style="stroke:#181818;stroke-width:0.5;"/></g><rect fill="#EEEEEE" height="3" style="stroke:#EEEEEE;stroke-width:1;" width="1117.1318" x="0" y="98.1436"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="98.1436" y2="98.1436"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="101.1436" y2="101.1436"/><rect fill="#EEEEEE" height="23.3105" style="stroke:#000000;stroke-width:2;" width="262.8281" x="427.1519" y="87.4883"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="244.5435" x="433.1519" y="104.0566">Node Startup: Device Spec Reporting</text><g class="message" data-entity-1="part4" data-entity-2="part3" id="msg1"><polygon fill="#181818" points="444.3555,168.7305,434.3555,172.7305,444.3555,176.7305,440.3555,172.7305" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="438.3555" x2="706.231" y1="172.7305" y2="172.7305"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="143.3491" x="498.6187" y="137.3672">Write Node Annotation</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="186.4878" x="477.0493" y="152.6777">hami.io/node-nvidia-register</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="217.8071" x="461.3896" y="167.9883">(UUID / VRAM / Compute / Model)</text></g><rect fill="#EEEEEE" height="3" style="stroke:#EEEEEE;stroke-width:1;" width="1117.1318" x="0" y="201.3857"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="201.3857" y2="201.3857"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="204.3857" y2="204.3857"/><rect fill="#EEEEEE" height="23.3105" style="stroke:#000000;stroke-width:2;" width="136.2241" x="490.4539" y="190.7305"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="117.9395" x="496.4539" y="207.2988">User Submits Pod</text><g class="message" data-entity-1="part1" data-entity-2="part2" id="msg2"><polygon fill="#181818" points="221.811,256.6621,231.811,260.6621,221.811,264.6621,225.811,260.6621" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="27.1826" x2="227.811" y1="260.6621" y2="260.6621"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="71.4746" x="94.7595" y="240.6094">Submit Pod</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="182.6284" x="39.1826" y="255.9199">(nvidia.com/gpu + gpumem)</text></g><g class="message" data-entity-1="part2" data-entity-2="part2" id="msg3"><line style="stroke:#181818;stroke-width:1;" x1="233.811" x2="275.811" y1="305.2832" y2="305.2832"/><line style="stroke:#181818;stroke-width:1;" x1="275.811" x2="275.811" y1="305.2832" y2="318.2832"/><line style="stroke:#181818;stroke-width:1;" x1="234.811" x2="275.811" y1="318.2832" y2="318.2832"/><polygon fill="#181818" points="244.811,314.2832,234.811,318.2832,244.811,322.2832,240.811,318.2832" style="stroke:#181818;stroke-width:1;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="148.1035" x="240.811" y="285.2305">Rewrite schedulerName</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="144.3838" x="242.6709" y="300.541">(force hami-scheduler)</text></g><g class="message" data-entity-1="part2" data-entity-2="part3" id="msg4"><polygon fill="#181818" points="421.3555,343.5938,431.3555,347.5938,421.3555,351.5938,425.3555,347.5938" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="233.811" x2="427.3555" y1="347.5938" y2="347.5938"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="175.5444" x="245.811" y="342.8516">Hand off to hami-scheduler</text></g><rect fill="#EEEEEE" height="3" style="stroke:#EEEEEE;stroke-width:1;" width="1117.1318" x="0" y="376.249"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="376.249" y2="376.249"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="379.249" y2="379.249"/><rect fill="#EEEEEE" height="23.3105" style="stroke:#000000;stroke-width:2;" width="153.2803" x="481.9258" y="365.5938"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="134.9956" x="487.9258" y="382.1621">Scheduling Decision</text><g class="message" data-entity-1="part3" data-entity-2="part3" id="msg5"><line style="stroke:#181818;stroke-width:1;" x1="433.3555" x2="475.3555" y1="435.5254" y2="435.5254"/><line style="stroke:#181818;stroke-width:1;" x1="475.3555" x2="475.3555" y1="435.5254" y2="448.5254"/><line style="stroke:#181818;stroke-width:1;" x1="434.3555" x2="475.3555" y1="448.5254" y2="448.5254"/><polygon fill="#181818" points="444.3555,444.5254,434.3555,448.5254,444.3555,452.5254,440.3555,448.5254" style="stroke:#181818;stroke-width:1;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="141.8701" x="472.8682" y="415.4727">Read Node Annotation</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="206.8955" x="440.3555" y="430.7832">Select target node and GPU UUID</text></g><g class="message" data-entity-1="part3" data-entity-2="part4" id="msg6"><polygon fill="#181818" points="695.231,504.457,705.231,508.457,695.231,512.457,699.231,508.457" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="433.3555" x2="701.231" y1="508.457" y2="508.457"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="133.688" x="503.4492" y="473.0938">Write Pod Annotation</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="203.1885" x="468.699" y="488.4043">hami.io/vgpu-devices-allocated</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="249.8755" x="445.3555" y="503.7148">(UUID + VRAM quota + compute quota)</text></g><rect fill="#EEEEEE" height="3" style="stroke:#EEEEEE;stroke-width:1;" width="1117.1318" x="0" y="537.1123"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="537.1123" y2="537.1123"/><line style="stroke:#000000;stroke-width:1;" x1="0" x2="1117.1318" y1="540.1123" y2="540.1123"/><rect fill="#EEEEEE" height="23.3105" style="stroke:#000000;stroke-width:2;" width="268.668" x="424.2319" y="526.457"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="250.3833" x="430.2319" y="543.0254">Device Allocation and Container Start</text><g class="message" data-entity-1="part4" data-entity-2="part4" id="msg7"><line style="stroke:#181818;stroke-width:1;" x1="707.231" x2="749.231" y1="596.3887" y2="596.3887"/><line style="stroke:#181818;stroke-width:1;" x1="749.231" x2="749.231" y1="596.3887" y2="609.3887"/><line style="stroke:#181818;stroke-width:1;" x1="708.231" x2="749.231" y1="609.3887" y2="609.3887"/><polygon fill="#181818" points="718.231,605.3887,708.231,609.3887,718.231,613.3887,714.231,609.3887" style="stroke:#181818;stroke-width:1;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="132.209" x="728.8115" y="576.3359">Read Pod Annotation</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="161.3701" x="714.231" y="591.6465">Get GPU UUID and quotas</text></g><g class="message" data-entity-1="part4" data-entity-2="part6" id="msg8"><polygon fill="#181818" points="1054.2021,634.6992,1064.2021,638.6992,1054.2021,642.6992,1058.2021,638.6992" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="707.231" x2="1060.2021" y1="638.6992" y2="638.6992"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="124.0205" x="824.7063" y="633.957">Mount /dev/nvidia*</text></g><path d="M529,651.6992 L529,737.6992 L884,737.6992 L884,661.6992 L874,651.6992 L529,651.6992" fill="#FEFFDD" style="stroke:#181818;stroke-width:0.5;"/><path d="M874,651.6992 L874,661.6992 L884,661.6992 L874,651.6992" fill="#FEFFDD" style="stroke:#181818;stroke-width:0.5;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="161.2178" x="535" y="669.2676">Allocate response injects:</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="181.0923" x="535" y="684.5781">&#8226; hostPath mount libvgpu.so</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="334.4644" x="535" y="699.8887">&#8226; hostPath mount ld.so.preload &#8594; /etc/ld.so.preload</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="264.7671" x="535" y="715.1992">&#8226; CUDA_DEVICE_MEMORY_LIMIT_&lt;index&gt;</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="164.7153" x="535" y="730.5098">&#8226; CUDA_DEVICE_SM_LIMIT</text><g class="message" data-entity-1="part5" data-entity-2="part6" id="msg9"><polygon fill="#181818" points="1054.2021,775.873,1064.2021,779.873,1054.2021,783.873,1058.2021,779.873" style="stroke:#181818;stroke-width:1;"/><line style="stroke:#181818;stroke-width:1;" x1="885.231" x2="1060.2021" y1="779.873" y2="779.873"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="86.1885" x="932.6223" y="759.8203">CUDA API call</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="156.9712" x="897.231" y="775.1309">(OOM if quota exceeded)</text></g><?plantuml-src VLJ1Rjf04BtxAqR99MbaV4SbaciWGOeJ12LHLIgqs8FPnjvrTjT0RZtr0_e9_RHyIMVNDe4awWdiVVFclNdZrOzSb4on0ZG-LIWIZ55hbc6Oyqn0WiAWylGUfL2NKkWMzGLpkGsLabimgaBF8vJX2I-PC70oqwXEW6cewC_nKLmPPhZ87d2vbl9nBhv7UQMf5qHI62Lpk67Fg7wuycqDWjCe6Wo6msEcQRB6jCfH-OETGP6Y-Z-PjlYFgVguuGd2EAyoBeZbLgO8OKOpE2H8Iriz74RHSVLr67C_aWfjV-hBk2058orpSYh58KL2A3WT3Xr9oWnRCermCbu_Qvwm7Ax6CuVBoihpUZrmGgQ6kbLb5ngPqn8Jc60fbHqaz7gUHzByoyPX5nuKDrZNXa98usJCnPeLlCDb8EZ05nkUSkOhpBWTs5oSpMQZFWHmFmbZ-ebaKLR44a1CyFpSoR6vmhHQ5jne6ClKzROPs-vjMDq6OG54MpVg9B88o1LSM6y55kTUYpygdU3MYTTjkBUieBcUhQGYvzQ1lpywecYzNpEHWbojmCWtSEUWMHexjpucNDjyh8cw_81dWYnzFy8fvfWOe40oD6072ROVnGPsVAyq54YRWvtJHp5iQ11-wYBLFijpcJ23QHl4HHt4KqLrz90qSRZdEenc6yAwqXgnKbuNqAtDmNOuISxMMqLNvAMru4XS5ztKsTIwj0KLxNf0UeCwqKyU4I383RLor8qGt5yLmCLFcfRkpiNBhx-mbjgCcLb3uQXolhGJw6ZvyN5A9vrIqJr3Ybz-_u40JH8SlQuBera_NFG7zwDei8W7yTtas-9c58--BZvpkXHsb-zHqxX6UEvpt5kq40Z78w0lCQSSxkvYuAic0jmbY2ccvzu_?></g></svg>