Deprecate --fast-stat-sync and replace with Criterion.logging_outputs_can_be_summed

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/980

Differential Revision: D19351116

Pulled By: myleott

fbshipit-source-id: a67b10637f53a80c37b0ce90eb27ced9709871db

Author: Myle Ott

fairseq/criterions/adaptive_loss.py

@@ -90,3 +90,12 @@ def aggregate_logging_outputs(logging_outputs):
         if sample_size != ntokens:
             agg_output['nll_loss'] = loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/cross_entropy.py

@@ -65,3 +65,12 @@ def aggregate_logging_outputs(logging_outputs):
         if sample_size != ntokens:
             agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/fairseq_criterion.py

@@ -37,3 +37,12 @@ def forward(self, model, sample, reduce=True):
     def aggregate_logging_outputs(logging_outputs):
         """Aggregate logging outputs from data parallel training."""
         raise NotImplementedError
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return False

fairseq/criterions/label_smoothed_cross_entropy.py

@@ -88,3 +88,12 @@ def aggregate_logging_outputs(logging_outputs):
             'nsentences': nsentences,
             'sample_size': sample_size,
         }
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py

@@ -88,3 +88,12 @@ def aggregate_logging_outputs(logging_outputs):
             'nsentences': nsentences,
             'sample_size': sample_size,
         }
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/legacy_masked_lm.py

@@ -145,3 +145,12 @@ def aggregate_logging_outputs(logging_outputs):
             'sample_size': sample_size,
         }
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/masked_lm.py

@@ -79,3 +79,12 @@ def aggregate_logging_outputs(logging_outputs):
             'sample_size': sample_size,
         }
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/nat_loss.py

@@ -167,3 +167,12 @@ def aggregate_logging_outputs(logging_outputs):
                 )
 
         return results
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/sentence_prediction.py

@@ -96,3 +96,12 @@ def aggregate_logging_outputs(logging_outputs):
         if sample_size != ntokens:
             agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True

fairseq/criterions/sentence_ranking.py

@@ -115,3 +115,12 @@ def aggregate_logging_outputs(logging_outputs):
         if sample_size != ntokens:
             agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)
         return agg_output
+
+    @staticmethod
+    def logging_outputs_can_be_summed() -> bool:
+        """
+        Whether the logging outputs returned by `forward` can be summed
+        across workers prior to calling `aggregate_logging_outputs`.
+        Setting this to True will improves distributed training speed.
+        """
+        return True