o ei 3 @sUddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z ddl mZddlmZddlmZmZddlmZmZddlmZmZdd lmZmZeeZej e!d <ed d Gd ddZ"ed d GdddZ#ed d GdddZ$GdddeZ%de&e'de e"fddZ(de&e'de e"fddZ)d5de'de'de*e&e'e e"ffddZ+d e'fd!d"Z,d#Z-d$Z.d%Z/Gd&d'd'Z0Gd(d)d)eZ1Gd*d+d+eZ2Gd,d-d-Z3d.e4d/e'dBd0e5d1e&e%d2e6e'ddf d3d4Z7dS)6N)ABCabstractmethod)CallableIterable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)get_world_sizetcpstore_clientloggerT)slotsc@s.eZdZUeed<eed<ddZddZdS)Response status_codetextcCs&|jdkrtd|jd|jdS)NzHTTP z: )r RuntimeErrorrselfrk/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/distributed/debug/_frontend.pyraise_for_status!s zResponse.raise_for_statuscCs t|jSN)jsonloadsrrrrrr% z Response.jsonN)__name__ __module__ __qualname__int__annotations__strrrrrrrrs  rc@seZdZUeed<eed<dS)NavLinkpathlabelN)r r!r"r%r$rrrrr&)s  r&c@s(eZdZUeed<edgefed<dS)Router'HTTPRequestHandlerhandlerN)r r!r"r%r$rbytesrrrrr)/s r)c@sneZdZedeefddZedeefddZde e e ffddZ de dBfd d Z de fd d Z dS) DebugHandlerreturncCdSrrrrrrroutes6zDebugHandler.routescCr/rrrrrr nav_links9r1zDebugHandler.nav_linkscCsiSrrrrrr templates<r1zDebugHandler.templatesNcCr/rrrrrrdump?r1zDebugHandler.dumpcCst|jSr)typer lowerrrrr dump_filenameBzDebugHandler.dump_filename)r r!r"rlistr)r0r&r2dictr%r3r4r7rrrrr-5sr-urlsr.cs^ddld}dtdtffdd }t|d}|||}Wd|S1s(wY|S)Nrurlr.cs|}t|j|jSr)postrrr)r=resprequestsrrgetQ zfetch_thread_pool..get) max_workers)rAr%rrmap)r;rDrBexecutorrespsrr@rfetch_thread_poolKs  rHcsNddldjdtdtfdddttdttffdd }t||S) Nrsessionr=r.c s^||4IdH}|IdH}t|j|WdIdHS1IdHs(wYdSr)r>rrstatus)rIr=r?rrrrfetch`s  0zfetch_aiohttp..fetchr;c sb4IdHtjfdd|DIdHWdIdHS1IdHs*wYdS)Ncsg|]}|qSrr).0r=)rKrIrr gz1fetch_aiohttp..gather..) ClientSessionasynciogather)r;aiohttprK)rIrrQes0zfetch_aiohttp..gather)rSrOr%rr9rrPrun)r;rQrrRr fetch_aiohttp[s rUendpointargscslt}ddttD}||}fdd|D}z t|}W||fSty5t|}Y||fSw)NcSsg|]}d|qS)rankr)rLrrrrrMnrNzfetch_all..cs$g|]}|ddqS)z /handler/?)decode)rLaddrrXrWrrrMps$)rranger multi_getrU ImportErrorrH)rWrXstorekeysaddrsrGrr^r fetch_allls   reblobcCst|}tj|ddS)N)indent)rrdumps)rfparsedrrr format_jsonzrCrkac {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} c @sLeZdZ ddeedededdfddZdd d Zdd d Z dd dZ dS)PeriodicDumperN@handlers output_dirinterval_secondsr.NcCs&||_||_||_t|_d|_dSr) _handlers _output_dir_interval_seconds threadingEvent _stop_event_thread)rrnrorprrr__init__s   zPeriodicDumper.__init__cCs2tj|jddtj|jddd|_|jdS)NT)exist_okz distributed.debug.PeriodicDumper)targetdaemonname)osmakedirsrrrtThread_runrwstartrrrrrszPeriodicDumper.startcCs&|j|jdur|jdSdSr)rvsetrwjoinrrrrstops  zPeriodicDumper.stopc Cs|jsx|jD]a}z|}Wnty!td|Yqw|dur'qt d}|d|d}t j |j |}zt|d }||Wdn1sTwYWqtyitd|Yqw|j|j|jrdSdS)NzFailed to dump %sz %Y%m%d_%H%M%S_z.txtwzFailed to write dump to %s)rvis_setrqr4 Exceptionr exceptionr7timestrftimer}r'rrropenwritewaitrs)rr+content timestampfilenamer'frrrr s.        zPeriodicDumper._run)rmr.N) r r!r"r9r-r%floatrxrrrrrrrrls rlc@s*eZdZUejZejed<dZe ed<dS)_IPv6HTTPServeraddress_familyirequest_queue_sizeN) r r!r"socketAF_INET6r AddressFamilyr$rr#rrrrr#s rc @s|eZdZUded<ddZddZdefdd Zdeee effd d Z defd d Z defdede de de fddZdS)r*FrontendServerfrontendcGstd|jd||dS)Nz%s %sr)rinfoclient_address)rformatrXrrr log_message+s zHTTPRequestHandler.log_messagecCs|j|dSr)r_handle_requestrrrrdo_GET2szHTTPRequestHandler.do_GETr.cCs t|jjSr)r r'rrrrget_path5rzHTTPRequestHandler.get_pathcCs t|Sr)r get_raw_queryrrrr get_query8rzHTTPRequestHandler.get_querycCs t|jjSr)r r'queryrrrrr;rz HTTPRequestHandler.get_raw_queryNr|defaultr5cCs$|}||vr |S|||dS)Nr)r)rr|rr5rrrr get_query_arg>sz HTTPRequestHandler.get_query_arg)r r!r"r$rrr%rr:r9rrobjectr5rrrrrr*(s" r*c@sfeZdZ ddedeedBfddZdddZdd d Zd e ddfd d Z de de de fddZdS)rNportrnc Cs|dur ddlm}|}ddd|D}tttd}|D] }||qt|}t |dd|_ |j j jt t t|d i|_|D]}|D] }|j|j|j<qJqD||_td tfd |i} d |f} t| | |_tj|jd ddd|_|jdS)Nr)default_handlers css4|]}|D]}d|jd|jdVqqdS)z z N)r2r'r()rLr+linkrrr Ss z*FrontendServer.__init__..)z base.htmlz raw_resp.htmlzjson_resp.htmlT)loader enable_async)ziprk enumerater2r*rrVrz distributed.debug.FrontendServer)rzrXr{r|)'torch.distributed.debug._debug_handlersrr BASE_TEMPLATERAW_RESP_TEMPLATEJSON_RESP_TEMPLATEupdater3r r _jinja_envglobalsrrkr_routesr0r+r'rqr5r*r_serverrtr_serverwr) rrrnrnav_html all_templatesr+rrouteRequestHandlerClassserver_addressrrrrxHsP    zFrontendServer.__init__r.cCs0z|jWdStytdYdSw)Nz got exception in frontend server)r serve_foreverrrrrrrrrs  zFrontendServer._servecCs|jdSr)rwrrrrrrr8zFrontendServer.joinreqc Cs|}||jvr|dd|dS|j|}z||}Wn&ttfyE}ztd||ddt|WYd}~dSd}~ww|d| dd| |j |dS) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) rr send_errorr SystemExitrrrepr send_response send_header end_headerswfiler)rrr'r+r?errrrs(     zFrontendServer._handle_requesttemplatekwargscKs|j|jdi|S)Nr)r get_templaterenderencode)rrrrrrrender_templateszFrontendServer.render_templaterr)r r!r"r#r9r-rxrrr*rr%rr,rrrrrrGs  = rrdump_dir dump_intervalrn enabled_dumpscsttjt||d}td|jjd}|dur2tfdd|D||}| td||z| W|durB| dSdS|durM| ww)N)rrnz"Frontend server started on port %dcsg|] }|vr|qSr)r7)rLr+rrrrMs  zmain..z2Periodic dumper started, writing to %s every %.0fs) rsetLevelloggingINFOrrr server_portrlrrr)rrrrnrserverdumperrrrmains2       r)rV)8rPrrr}rrtrabcrrcollections.abcrrconcurrent.futuresr dataclassesr http.serverrr urllib.parser r jinja2r r torch.distributed.debug._storerr getLoggerr rLoggerr$rr&r)r-r9r%rHrUtuplererkrrrrlrr*rr#rrrrrrrs^    ( @5b