Nj<^SSKJr SSKJr SSKJr SSKJrJrJrJ r J r J r SSK J r SSKJr SSKJr SSKJrJrJrJrJr \(aSS KJrJr S rS rS rS rSr"SS\5r"SS\5r "SS\5r!"SS\5r""SS\5r#"SS\5r$"SS\5r%SSjr&g) ) annotations) TYPE_CHECKING)tabulate) DebugHandler fetch_all format_jsonNavLinkResponseRoute)tcpstore_client)build_db) JobConfig) CollectiveDatabaseGroup MembershipNCCLCall)FrontendServerHTTPRequestHandlerz {% extends "base.html" %} {% block header %}

{% block title %}Index{% endblock %}

{% endblock %} {% block content %} Hi {% endblock %} aQ {% extends "base.html" %} {% block header %}

{% block title %}py-spy Stack Traces{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} ag {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %}

Groups

{{ groups | safe }}

Memberships

{{ memberships | safe }}

Collectives

{{ collectives | safe }}

NCCL Calls

{{ ncclcalls | safe }} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}torch.profiler{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} {% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}TCPStore Keys{% endblock %}

{% endblock %} {% block content %} 
    {% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
    {% endfor %}
{% endblock %} c<\rSrSrSSjrS SjrS SjrS SjrSrg) IndexHandlerc0[SUR5/$)N/r _handleselfs x/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/torch/distributed/debug/_debug_handlers.pyroutesIndexHandler.routessc4<<())c[SS5/$)NrHomer rs r nav_linksIndexHandler.nav_linkssV$%%r"cS[0$Nz index.html)INDEX_TEMPLATErs r templatesIndexHandler.templatess n--r"c8URRS5$r))frontendrender_template)rreqs rrIndexHandler._handles||++L99r"Nreturnz list[Route]r4z list[NavLink]r4zdict[str, str]r0rr4bytes __name__ __module__ __qualname____firstlineno__r r&r+r__static_attributes__r2r"rrrs*&.:r"rcF\rSrSrS SjrS SjrS SjrS SjrS SjrSr g) StacksHandlerc0[SUR5/$)N/stacksrrs rr StacksHandler.routessi.//r"c[SS5/$)NrCzPython Stack Tracesr%rs rr&StacksHandler.nav_linkss #89::r"cR[S5up#URRSSX#S9$)Ndump_tracebackz raw_resp.htmlStackstitleaddrsrespsrr.r/rr0rLrMs rrStacksHandler._handles2 !12 ||++ 85,  r"c&[S5up/n[[X55HZunupVURSUSUS35 URURS:Xa UR OSUR35 M\ SR U5$)NrH === Rank :  ===Error:  r enumeratezipappend status_codetextjoinrrLrMpartsiaddrresps rdumpStacksHandler.dumps !12 (U):;OA| LL9QCr$t4 5 LL!--4 GDDTDTCU:V  < yyr"cg)Nstacksr2rs r dump_filenameStacksHandler.dump_filenamesr"r2Nr3r5r7r4z str | Noner4str r:r;r<r=r r&rrdrhr>r2r"rr@r@s0;  r"r@cP\rSrSrS SjrS SjrS SjrS SjrSSjrSSjr Sr g ) PySpyHandlerc0[SUR5/$)N /pyspy_dumprrs rr PySpyHandler.routessmT\\233r"c[SS5/$)Nrrz py-spy Stacksr%rs rr&PySpyHandler.nav_linkss 788r"cS[0$)Npyspy_dump.html)PYSPY_DUMP_TEMPLATErs rr+PySpyHandler.templatess!#677r"cp[SUR55up#URRSUUS9$)N pyspy_dumprwrLrM)r get_raw_queryr.r/rOs rrPySpyHandler._handles> s/@/@/BC ||++ ,  r"c&[S5up/n[[X55HZunupVURSUSUS35 URURS:Xa UR OSUR35 M\ SR U5$)Nr{rRrSrTrUrVrWrXr_s rrdPySpyHandler.dumps . (U):;OA| LL9QCr$t4 5 LL!--4 GDDTDTCU:V  < yyr"cg)Nr{r2rs rrhPySpyHandler.dump_filenamesr"r2Nr3r5r6r7rjrk r:r;r<r=r r&r+rrdrhr>r2r"rroros 498  r"roc\rSrSrSSjrSSjrSSjr\SSj5rSSjr SSjr SSjr SS jr SS jr SS jrSS jrS rg)FlightRecorderHandlerc[SUR5[SUR5[SUR5[SUR5/$)N /fr_trace/fr_trace_json/fr_trace_nccl/fr_trace_nccl_json)r _handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_jsonrs rr FlightRecorderHandler.routessO +t44 5 "D$>$> ? "D$>$> ? ')H)H I   r"c^[SS5[SS5[SS5[SS5/$)NrzFlightRecorder CPUrz(JSON)rFlightRecorder NCCLrr%rs rr&FlightRecorderHandler.nav_linkss9 K!5 6 $h / $&; < )8 4   r"cS[0$)N fr_trace.html)FR_TRACE_TEMPLATErs rr+FlightRecorderHandler.templates!233r"cR[5nUR/S9nSUlSUl0n[ U5H?upVUR 5 UXS.UR 5EnSU;a/US'XtSUS3'MA [[UR555Sn[XCU5$)N)argsT)rank host_nameentriesrz.jsonversion) r parse_argsallow_incomplete_ranksverboserYraise_for_statusjsonnextitervaluesr ) rLrMconfigrdetailsrrcrdrs r _build_dbFlightRecorderHandler._build_db s  b )&*# #E*JD  ! ! #"[))+D $"$Y*.d4&& '+tGNN,-.y9w//r"c TURX#5nURSS[UR[R SS9[UR [R SS9[UR[R SS9[UR[R SS9S9$)NrFlightRecorderhtmlheaderstablefmt)rKgroups memberships collectives ncclcalls) rr/rrr_fieldsrrrrrr)rserverrLrMdbs r_render_tables$FlightRecorderHandler._render_tables!s^^E )%% "BIIu}}vN  (:(:V! (:(:Vr||X5E5EPVW&  r"ch[S5up#URURU[U55$)N fr_trace_jsonrrr.listrOs rr&FlightRecorderHandler._handle_fr_trace2s+ 1 ""3<<U DDr"cT[S5up#URRSSUUS9$)Nrjson_resp.htmlrrJrNrOs rr+FlightRecorderHandler._handle_fr_trace_json6s6 1 ||++ " ,  r"cj[SS5up#URURU[U55$)Ndump_nccl_trace_jsononlyactive=truerrOs rr+FlightRecorderHandler._handle_fr_trace_nccl?s/ !79JK ""3<<U DDr"cV[SS5up#URRSSUUS9$)NrrrrrJrNrOs rr0FlightRecorderHandler._handle_fr_trace_nccl_jsonCs: !79JK ||++ ' ,  r"c/n[S5up#URU[U55nURSS[ UR [ RSS9S[ UR[RSS9S[ UR[RSS9S[ UR[RSS9/ 5 [S S 5upVURU[U55nURS S S[ UR [ RSS9S[ UR[RSS9S[ UR[RSS9S[ UR[RSS9/ 5 SR!U5$![a URS 5 N.f=f)Nrz=== FR Trace ===z--- Groups ---plainrz--- Memberships ---z--- Collectives ---z--- NCCL Calls ---rrz=== FR Trace NCCL ===z1 === FR Trace NCCL === Failed to fetch NCCL tracerW)rrrextendrrrrrrrrrr Exceptionr[r^)rr`rLrMr nccl_addrs nccl_respsnccl_dbs rrdFlightRecorderHandler.dumpLs 1  ^^E4; / " EMMGL%1C1CgV%1C1CgV$x/?/?'R   P%.&(9& "JnnZj1ABG LL+$W^^U]]WU)++ * 2 2!( *++ * 2 2!( ))) ( 0 0!(% :yy P LLN O Ps CF##GGcg)Nfr_tracer2rs rrh#FlightRecorderHandler.dump_filenamer"r2Nr3r5r6)rL list[str]rMlist[Response]r4r)rrrLrrMrr4r8r7rjrk)r:r;r<r=r r&r+ staticmethodrrrrrrrdrhr>r2r"rrrsk  400, $ -6 ?M  "E E 5 nr"rc<\rSrSrSSjrS SjrS SjrS SjrSrg) ProfilerHandleric0[SUR5/$)N/profilerrs rr ProfilerHandler.routessj$,,/00r"c[SS5/$)Nrztorch profilerr%rs rr&ProfilerHandler.nav_linkss $4566r"cS[0$)N profile.html)PROFILE_TEMPLATErs rr+ProfilerHandler.templatess 011r"cURSS[S9n[SSU35up4URR SX4S9$)Ndurationg?)defaulttype torch_profilez duration=rr|) get_query_argfloatrr.r/)rr0rrLrMs rrProfilerHandler._handlesH$$Z5$I IhZ2HI ||++N%+UUr"r2Nr3r5r6r7r9r2r"rrrs172Vr"rcF\rSrSrS SjrS SjrS SjrS SjrS SjrSr g)WaitCountersHandleric0[SUR5/$)N/wait_countersrrs rr WaitCountersHandler.routess& 566r"c[SS5/$)Nr Wait Countersr%rs rr&WaitCountersHandler.nav_linkss(/:;;r"cR[S5up#URRSSX#S9$)Nwait_counter_valuesrrrJrNrOs rrWaitCountersHandler._handles2 !67 ||++ O5,  r"cZ[S5up/n[[X55HtunupVURSUSUS35 URS:Xa&UR[ UR 55 MVURSUR35 Mv SRU5$)NrrRrSrTrUrVrW)rrYrZr[r\rr]r^r_s rrdWaitCountersHandler.dumps !67 (U):;OA| LL9QCr$t4 53& [34 wt'7'7&89: < yyr"cg)N wait_countersr2rs rrh!WaitCountersHandler.dump_filenamesr"r2Nr3r5r7rjrkrmr2r"rrrs7<  r"rcP\rSrSrS SjrS SjrS SjrS SjrSSjrSSjr Sr g )TCPStoreHandleric0[SUR5/$)N /tcpstorerrs rr TCPStoreHandler.routessk4<<011r"c[SS5/$)NrTCPStorer%rs rr&TCPStoreHandler.nav_linkss Z011r"cS[0$)N tcpstore.html)TCPSTORE_TEMPLATErs rr+TCPStoreHandler.templatesrr"c[SS9nUR5nUR5 URU5Vs/sHn[ U5PM nnUR R SX5S9$s snf)Nrprefixr)keysr)r list_keyssort multi_getreprr.r/)rr0storer vrs rrTCPStoreHandler._handlescr*  #(??4#89#8a$q'#89||++O$+VV:sA.c([SS9nUR5nUR5 URU5Vs/sHn[ U5PM nn[ X$5VVs/sH upSUSU3PM nnnSR U5$s snfs snnf)NrrrSrW)r r r r rrZr^)rrr rrkr`s rrdTCPStoreHandler.dumpsr*  #(??4#89#8a$q'#89),T):;):A3b):;yy:;s B "Bcg)Ntcpstorer2rs rrhTCPStoreHandler.dump_filenamerr"r2Nr3r5r6r7rjrkrr2r"rrrs!224W r"rc[5[5[5[5[ 5[ 5[ 5/$)N)rr@rorrrrr2r"rdefault_handlersrs3 r"N)r4zlist[DebugHandler])' __future__rtypingrr!torch.distributed.debug._frontendrrrr r r torch.distributed.debug._storer 4torch.distributed.flight_recorder.components.builderr ;torch.distributed.flight_recorder.components.config_managerr2torch.distributed.flight_recorder.components.typesrrrrrrrr*rxrrrrr@rorrrrrr2r"rr!s" ;IQT4"?B ( :< :L6<@NLNb Vl V ,8l: r"