Merge pull request #6 from cpanel/RE-937

Fix race condition that can occur between ssh_retry and reboot_watch
cpanel · Nov 4, 2024 · aff273e · aff273e
2 parents be0b30c + c97d891
commit aff273e
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 27 deletions.
diff --git a/.github/workflows/openstack.yml b/.github/workflows/openstack.yml
@@ -111,7 +111,7 @@ jobs:
 
         - name: Checkout Repo and Commit
           if: github.event_name != 'pull_request'
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -129,7 +129,7 @@ jobs:
 
         - name: Checking out Repo and Pull Request
           if: github.event_name == 'pull_request'
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -168,7 +168,7 @@ jobs:
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
 
         - name: Setup for Integration Checks Prior to Running Elevate
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -203,7 +203,7 @@ jobs:
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
 
         - name: Starting Elevate
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -270,7 +270,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Monitor Elevate for Reboot from Stage 1 into Stage 2
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -335,7 +335,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Monitor Elevate for Stage 3 Reboot
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -400,7 +400,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Monitor Elevate for Stage 4 Reboot
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -464,7 +464,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Monitor Elevate for Stage 5 Reboot
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -528,7 +528,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Monitor Elevate for Final Reboot
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -592,7 +592,7 @@ jobs:
             VM_IP: ${{ steps.VM_IP.outputs.VM_IP }}
           run: echo "VM_IP is ${{ steps.VM_IP.outputs.VM_IP }}"
         - name: Verify End Result Integration Tests
-          uses: appleboy/ssh-action@v1.0.3
+          uses: appleboy/ssh-action@v1.1.0
           with:
             host: ${{ steps.VM_IP.outputs.VM_IP }}
             username: 'root'
@@ -617,4 +617,4 @@ jobs:
           name: ${{ github.run_id }}-tf.out.json
           path: ${{ github.workspace }}/tf.out.json
       - name: Destroy OpenStack VM
-        run: terraform destroy -no-color -auto-approve 
+        run: terraform destroy -no-color -auto-approve
diff --git a/.github/workflows/openstack/ssh_retry b/.github/workflows/openstack/ssh_retry
@@ -6,7 +6,7 @@ my $HOST    = $ARGV[0];
 my $PORT    = $ARGV[1] // 22;
 my $RETVAL  = 1;
 my $RETRIES = 0;
-my $RETRY   = $ARGV[2] // 240;
+my $RETRY   = $ARGV[2] // 900;
 
 return unless defined $HOST;
 
@@ -32,5 +32,5 @@ while ( $RETVAL != 0 ) {
         print "## [$time] [ERROR]: ssh_retry.pl: MAX_RETRIES has been reached.\n";
         exit 1;
     }
-    sleep 5;
+    sleep 1;
 }
diff --git a/elevate-cpanel b/elevate-cpanel
@@ -6717,8 +6717,8 @@ EOS
         return instance()->update_with_options( $options, $pkgs );
     }
 
-    sub update_allow_erasing (@args) {
-        return instance()->update_allow_erasing(@args);
+    sub update_allow_erasing (@additional_args) {
+        return instance()->update_allow_erasing(@additional_args);
     }
 
     sub makecache () {
@@ -6893,7 +6893,7 @@ EOS
         die "update_with_options unimplemented";
     }
 
-    sub update_allow_erasing ( $self, @args ) {
+    sub update_allow_erasing ( $self, @additional_args ) {
         die "update_allow_erasing unimplemented";
     }
 
@@ -7153,13 +7153,16 @@ EOS
         return;
     }
 
-    sub update_allow_erasing ( $self, @args ) {
+    sub update_allow_erasing ( $self, @additional_args ) {
         my $pkgmgr = $self->_pkgmgr();
 
-        my @additional_args = scalar @args ? @args : '';
-
-        $self->ssystem( $pkgmgr, '-y', '--allowerasing', @additional_args, 'update' );
+        my @args = (
+            '-y',
+            '--allowerasing',
+        );
 
+        push @args, @additional_args;
+        $self->ssystem_and_die( $pkgmgr, @args, 'update' );
         return;
     }
 

diff --git a/lib/Elevate/PkgMgr.pm b/lib/Elevate/PkgMgr.pm
@@ -125,8 +125,8 @@ sub update_with_options ( $options, $pkgs ) {
     return instance()->update_with_options( $options, $pkgs );
 }
 
-sub update_allow_erasing (@args) {
-    return instance()->update_allow_erasing(@args);
+sub update_allow_erasing (@additional_args) {
+    return instance()->update_allow_erasing(@additional_args);
 }
 
 sub makecache () {

diff --git a/lib/Elevate/PkgMgr/Base.pm b/lib/Elevate/PkgMgr/Base.pm
@@ -156,7 +156,7 @@ sub update_with_options ( $self, $options, $pkgs ) {
     die "update_with_options unimplemented";
 }
 
-sub update_allow_erasing ( $self, @args ) {
+sub update_allow_erasing ( $self, @additional_args ) {
     die "update_allow_erasing unimplemented";
 }
 

diff --git a/lib/Elevate/PkgMgr/YUM.pm b/lib/Elevate/PkgMgr/YUM.pm
@@ -254,13 +254,16 @@ sub update_with_options ( $self, $options, $pkgs ) {
     return;
 }
 
-sub update_allow_erasing ( $self, @args ) {
+sub update_allow_erasing ( $self, @additional_args ) {
     my $pkgmgr = $self->_pkgmgr();
 
-    my @additional_args = scalar @args ? @args : '';
-
-    $self->ssystem( $pkgmgr, '-y', '--allowerasing', @additional_args, 'update' );
+    my @args = (
+        '-y',
+        '--allowerasing',
+    );
 
+    push @args, @additional_args;
+    $self->ssystem_and_die( $pkgmgr, @args, 'update' );
     return;
 }