Fix storage logic (microsoft#2488)

CHNxindong · May 25, 2020 · e640ad6 · e640ad6
1 parent 8d8fcd2
commit e640ad6
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 19 deletions.
diff --git a/src/nni_manager/training_service/kubernetes/kubernetesData.ts b/src/nni_manager/training_service/kubernetes/kubernetesData.ts
@@ -47,10 +47,10 @@ export NNI_EXP_ID={4}
 export NNI_CODE_DIR={5}
 export NNI_TRIAL_SEQ_ID={6}
 {7}
-mkdir -p $NNI_SYS_DIR
+mkdir -p $NNI_SYS_DIR/code
 mkdir -p $NNI_OUTPUT_DIR
-cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR
-cd $NNI_SYS_DIR
-sh install_nni.sh
+cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code
+sh $NNI_SYS_DIR/install_nni.sh
+cd $NNI_SYS_DIR/code
 python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \
 --nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr`;
diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts
@@ -477,16 +477,14 @@ class LocalTrainingService implements TrainingService {
     private getScript(localTrialConfig: TrialConfig, workingDirectory: string): string[] {
         const script: string[] = [];
         if (process.platform === 'win32') {
-            script.push(`Copy-Item $env:NNI_CODE_DIR\\* -Destination $env:NNI_SYS_DIR -Recurse`);
-            script.push(`cd $env:NNI_SYS_DIR`);
+            script.push(`cd $env:NNI_CODE_DIR`);
             script.push(
                 `cmd.exe /c ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`,
                 `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
                 `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
                 `Write $LASTEXITCODE " " $NOW_DATE  | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
         } else {
-            script.push(`cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR`);
-            script.push(`cd $NNI_SYS_DIR`);
+            script.push(`cd $NNI_CODE_DIR`);
             script.push(`eval ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`);
             if (process.platform === 'darwin') {
                 // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x

diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts
@@ -31,6 +31,6 @@ fi`;
 
 export const PAI_K8S_TRIAL_COMMAND_FORMAT: string =
 `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \
-&& NNI_CODE_DIR={6} && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR && cd $NNI_SYS_DIR && sh install_nni.sh \
-&& python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \
+&& NNI_CODE_DIR={6} && mkdir -p $NNI_SYS_DIR/code && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code && sh $NNI_SYS_DIR/install_nni.sh \
+&& cd $NNI_SYS_DIR/code && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \
 --nni_manager_version '{10}' --log_collection '{11}'`;
diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts
@@ -290,8 +290,6 @@ class PAIK8STrainingService extends PAITrainingService {
             await this.writeParameterFile(trialJobDetail.logPath, trialJobDetail.form.hyperParameters);
         }
 
-        //Copy codeDir files to local working folder
-        await execCopydir(this.paiTrialConfig.codeDir, trialJobDetail.logPath);
         //Generate Job Configuration in yaml format
         const paiJobConfig = this.generateJobConfigInYamlFormat(trialJobDetail);
         this.log.debug(paiJobConfig);

diff --git a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
@@ -22,10 +22,10 @@ class LinuxCommands extends OsCommands {
             export NNI_PLATFORM=remote NNI_SYS_DIR=${workingDirectory} NNI_OUTPUT_DIR=${workingDirectory} NNI_TRIAL_JOB_ID=${trialJobId} \
             NNI_EXP_ID=${experimentId} NNI_TRIAL_SEQ_ID=${trialSequenceId} NNI_CODE_DIR=${codeDir}
             export MULTI_PHASE=${isMultiPhase}
-
-            cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR
-            cd $NNI_SYS_DIR
-            sh install_nni.sh
+            mkdir -p $NNI_SYS_DIR/code
+            cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code
+            sh $NNI_SYS_DIR/install_nni.sh
+            cd $NNI_SYS_DIR/code
             python3 -m nni_trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \
                 --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \
                 --job_id_file ${jobIdFileName} \

diff --git a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts
@@ -28,9 +28,9 @@ class WindowsCommands extends OsCommands {
             set MULTI_PHASE=${isMultiPhase}
             set NNI_CODE_DIR=${codeDir}
             ${cudaVisibleSetting !== "" ? "set " + cudaVisibleSetting : ""}
-
-            robocopy /s %NNI_CODE_DIR%/. %NNI_SYS_DIR%
-            cd %NNI_SYS_DIR%
+            md %NNI_SYS_DIR%/code
+            robocopy /s %NNI_CODE_DIR%/. %NNI_SYS_DIR%/code
+            cd %NNI_SYS_DIR%/code
             python -c "import nni" 2>nul
             if not %ERRORLEVEL% EQU 0 (
                 echo installing NNI as exit code of "import nni" is %ERRORLEVEL%