fix dataloader initialization to only happen once. Not on every epoch. (

#143)

dlio_benchmark/main.py

@@ -249,7 +249,6 @@ def _train(self, epoch):
         Training loop for reading the dataset and performing training computations.
         :return: returns total steps.
         """
-        self.args.reconfigure(epoch, DatasetType.TRAIN)
         block = 1  # A continuous period of training steps, ended by checkpointing
         block_step = overall_step = 1  # Steps are taken within blocks
         max_steps = math.floor(self.num_samples * self.num_files_train / self.batch_size / self.comm_size)
@@ -258,7 +257,6 @@ def _train(self, epoch):
         self.stats.start_block(epoch, block)
 
         loader = self.framework.get_loader(dataset_type=DatasetType.TRAIN)
-        loader.read()
         t0 = time()
         for batch in dlp.iter(loader.next()):
             self.stats.batch_loaded(epoch, overall_step, block, t0)
@@ -329,13 +327,15 @@ def run(self):
             # Keep track of the next epoch at which we will evaluate
             next_eval_epoch = self.eval_after_epoch
             self.next_checkpoint_epoch = self.checkpoint_after_epoch
-
+            epoch = 1
+            self.args.reconfigure(epoch, DatasetType.TRAIN)
+            # Initialize the dataset
+            self.framework.init_loader(self.args.format, epoch=epoch, data_loader=self.args.data_loader)
+            loader = self.framework.get_loader(dataset_type=DatasetType.TRAIN)
+            loader.read()
             for epoch in range(1, self.epochs + 1):
                 self.next_checkpoint_step = self.steps_between_checkpoints
                 self.stats.start_train(epoch)
-
-                # Initialize the dataset
-                self.framework.init_loader(self.args.format, epoch=epoch, data_loader=self.args.data_loader)
                 steps = self._train(epoch)
                 self.stats.end_train(epoch, steps)
                 logging.debug(f"{utcnow()} Rank {self.my_rank} returned after {steps} steps.")