128
128
129
129
130
130
< div class ="version ">
131
- < a href ='http://pytorch.org/docs/versions.html '> 1.4.0a0+6e0312a ▼</ a >
131
+ < a href ='http://pytorch.org/docs/versions.html '> 1.4.0a0+e7b6ea5 ▼</ a >
132
132
</ div >
133
133
134
134
@@ -292,7 +292,6 @@ <h1>Source code for torch.distributed</h1><div class="highlight"><pre>
292
292
< span > </ span > < span class ="kn "> from</ span > < span class ="nn "> __future__</ span > < span class ="k "> import</ span > < span class ="n "> absolute_import</ span > < span class ="p "> ,</ span > < span class ="n "> division</ span > < span class ="p "> ,</ span > < span class ="n "> print_function</ span > < span class ="p "> ,</ span > < span class ="n "> unicode_literals</ span >
293
293
294
294
< span class ="kn "> import</ span > < span class ="nn "> torch</ span >
295
- < span class ="kn "> import</ span > < span class ="nn "> sys</ span >
296
295
297
296
298
297
< span class ="k "> def</ span > < span class ="nf "> is_available</ span > < span class ="p "> ():</ span >
@@ -310,42 +309,6 @@ <h1>Source code for torch.distributed</h1><div class="highlight"><pre>
310
309
< span class ="c1 "> # See the comment in `distributed_c10d.py` above `_backend` on why we expose</ span >
311
310
< span class ="c1 "> # this.</ span >
312
311
< span class ="kn "> from</ span > < span class ="nn "> .distributed_c10d</ span > < span class ="k "> import</ span > < span class ="n "> _backend</ span > < span class ="c1 "> # noqa: F401</ span >
313
- < span class ="k "> if</ span > < span class ="n "> sys</ span > < span class ="o "> .</ span > < span class ="n "> version_info</ span > < span class ="o "> >=</ span > < span class ="p "> (</ span > < span class ="mi "> 3</ span > < span class ="p "> ,</ span > < span class ="mi "> 0</ span > < span class ="p "> ):</ span >
314
- < span class ="kn "> from</ span > < span class ="nn "> .rpc_api</ span > < span class ="k "> import</ span > < span class ="n "> _init_rpc</ span >
315
- < span class ="kn "> from</ span > < span class ="nn "> .rpc_api</ span > < span class ="k "> import</ span > < span class ="o "> *</ span > < span class ="c1 "> # noqa: F401</ span >
316
-
317
- < span class ="k "> def</ span > < span class ="nf "> init_model_parallel</ span > < span class ="p "> (</ span > < span class ="n "> self_name</ span > < span class ="p "> ,</ span >
318
- < span class ="n "> backend</ span > < span class ="o "> =</ span > < span class ="n "> RpcBackend</ span > < span class ="o "> .</ span > < span class ="n "> PROCESS_GROUP</ span > < span class ="p "> ,</ span >
319
- < span class ="n "> self_rank</ span > < span class ="o "> =-</ span > < span class ="mi "> 1</ span > < span class ="p "> ,</ span >
320
- < span class ="n "> init_method</ span > < span class ="o "> =</ span > < span class ="kc "> None</ span > < span class ="p "> ,</ span >
321
- < span class ="n "> num_send_recv_threads</ span > < span class ="o "> =</ span > < span class ="mi "> 4</ span > < span class ="p "> ):</ span >
322
- < span class ="sa "> r</ span > < span class ="sd "> """</ span >
323
- < span class ="sd "> Initializes model parallel primitives such as the local rpc agent</ span >
324
- < span class ="sd "> and distributed autograd.</ span >
325
-
326
- < span class ="sd "> Initializes the local RPC agent which immediately makes the current</ span >
327
- < span class ="sd "> process ready to send and receive RPCs. The caller needs to make</ span >
328
- < span class ="sd "> sure the specified backend is properly intialized before calling</ span >
329
- < span class ="sd "> this method. For example, to use ``pg`` (ProcessGroup) backend,</ span >
330
- < span class ="sd "> ``init_process_group`` must be invoked prior to this method.</ span >
331
-
332
- < span class ="sd "> Arguments:</ span >
333
- < span class ="sd "> backend (Enum): type of RPC backend implementation.</ span >
334
- < span class ="sd "> Currently, process group backend is the only</ span >
335
- < span class ="sd "> available backend implementation. (default:</ span >
336
- < span class ="sd "> ``RpcBackend.PROCESS_GROUP``).</ span >
337
- < span class ="sd "> self_name (str): a globally unique name of this node. (e.g.,</ span >
338
- < span class ="sd "> ``Trainer3``, ``ParameterServer2``, ``Master``,</ span >
339
- < span class ="sd "> ``Worker1``) Name can only contain number, alphabet,</ span >
340
- < span class ="sd "> underscore, and/or dash, and must be shorter than</ span >
341
- < span class ="sd "> 128 characters.</ span >
342
- < span class ="sd "> self_rank (int): a globally unique id/rank of this node.</ span >
343
- < span class ="sd "> init_method(str): backend specific init arguments.</ span >
344
- < span class ="sd "> num_send_recv_threads(int): Number of threads for send/recv work.</ span >
345
- < span class ="sd "> """</ span >
346
- < span class ="n "> _init_rpc</ span > < span class ="p "> (</ span > < span class ="n "> backend</ span > < span class ="p "> ,</ span > < span class ="n "> self_name</ span > < span class ="p "> ,</ span > < span class ="n "> self_rank</ span > < span class ="p "> ,</ span > < span class ="n "> init_method</ span > < span class ="p "> ,</ span > < span class ="n "> num_send_recv_threads</ span > < span class ="p "> )</ span >
347
- < span class ="kn "> from</ span > < span class ="nn "> .rpc_api</ span > < span class ="k "> import</ span > < span class ="n "> _agent</ span >
348
- < span class ="n "> autograd</ span > < span class ="o "> .</ span > < span class ="n "> _init</ span > < span class ="p "> (</ span > < span class ="n "> _agent</ span > < span class ="o "> .</ span > < span class ="n "> get_worker_info</ span > < span class ="p "> ()</ span > < span class ="o "> .</ span > < span class ="n "> id</ span > < span class ="p "> )</ span >
349
312
</ pre > </ div >
350
313
351
314
</ article >
0 commit comments