@@ -644,7 +644,144 @@ func TestSingleNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
644644 t .Logf ("%s: running airgap update" , time .Now ().Format (time .RFC3339 ))
645645 line = []string {"airgap-update.sh" }
646646 if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
647- t .Fatalf ("fail to run kots upstream upgrade: %v" , err )
647+ t .Fatalf ("fail to run airgap update: %v" , err )
648+ }
649+ // remove the airgap bundle after upgrade
650+ line = []string {"rm" , "/tmp/upgrade/release.airgap" }
651+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
652+ t .Fatalf ("fail to remove airgap bundle on node %s: %v" , tc .Nodes [0 ], err )
653+ }
654+
655+ if _ , _ , err := runTestimTest (t , tc , "deploy-airgap-upgrade" ); err != nil {
656+ t .Fatalf ("fail to run testim test deploy-airgap-upgrade: %v" , err )
657+ }
658+
659+ t .Logf ("%s: checking installation state after upgrade" , time .Now ().Format (time .RFC3339 ))
660+ line = []string {"check-postupgrade-state.sh" }
661+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
662+ t .Fatalf ("fail to check postupgrade state: %v" , err )
663+ }
664+
665+ t .Logf ("%s: test complete" , time .Now ().Format (time .RFC3339 ))
666+ }
667+
668+ func TestMultiNodeAirgapUpgradeUbuntuJammy (t * testing.T ) {
669+ t .Parallel ()
670+
671+ t .Logf ("%s: downloading airgap files" , time .Now ().Format (time .RFC3339 ))
672+ airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
673+ airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
674+ wg := sync.WaitGroup {}
675+ wg .Add (2 )
676+ go func () {
677+ downloadAirgapBundle (t , fmt .Sprintf ("appver-%s" , os .Getenv ("SHORT_SHA" )), airgapInstallBundlePath )
678+ wg .Done ()
679+ }()
680+ go func () {
681+ downloadAirgapBundle (t , fmt .Sprintf ("appver-%s-upgrade" , os .Getenv ("SHORT_SHA" )), airgapUpgradeBundlePath )
682+ wg .Done ()
683+ }()
684+ wg .Wait ()
685+
686+ tc := cluster .NewTestCluster (& cluster.Input {
687+ T : t ,
688+ Nodes : 2 ,
689+ Image : "ubuntu/jammy" ,
690+ WithProxy : true ,
691+ AirgapInstallBundlePath : airgapInstallBundlePath ,
692+ AirgapUpgradeBundlePath : airgapUpgradeBundlePath ,
693+ })
694+ defer func () {
695+ if t .Failed () {
696+ generateAndCopySupportBundle (t , tc )
697+ }
698+ tc .Destroy ()
699+ }()
700+
701+ // delete airgap bundles once they've been copied to the nodes
702+ if err := os .Remove (airgapInstallBundlePath ); err != nil {
703+ t .Logf ("failed to remove airgap install bundle: %v" , err )
704+ }
705+ if err := os .Remove (airgapUpgradeBundlePath ); err != nil {
706+ t .Logf ("failed to remove airgap upgrade bundle: %v" , err )
707+ }
708+
709+ // upgrade airgap bundle is only needed on the first node
710+ line := []string {"rm" , "/tmp/ec-release-upgrade.tgz" }
711+ if _ , _ , err := RunCommandOnNode (t , tc , 1 , line ); err != nil {
712+ t .Fatalf ("fail to remove upgrade airgap bundle on node %s: %v" , tc .Nodes [1 ], err )
713+ }
714+
715+ t .Logf ("%s: preparing embedded cluster airgap files on node 0" , time .Now ().Format (time .RFC3339 ))
716+ line = []string {"airgap-prepare.sh" }
717+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
718+ t .Fatalf ("fail to prepare airgap files on node %s: %v" , tc .Nodes [0 ], err )
719+ }
720+
721+ t .Logf ("%s: installing embedded-cluster on node 0" , time .Now ().Format (time .RFC3339 ))
722+ line = []string {"single-node-airgap-install.sh" }
723+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
724+ t .Fatalf ("fail to install embedded-cluster on node %s: %v" , tc .Nodes [0 ], err )
725+ }
726+ // remove the airgap bundle after installation
727+ line = []string {"rm" , "/tmp/release.airgap" }
728+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
729+ t .Fatalf ("fail to remove airgap bundle on node %s: %v" , tc .Nodes [0 ], err )
730+ }
731+
732+ if err := setupTestim (t , tc ); err != nil {
733+ t .Fatalf ("fail to setup testim: %v" , err )
734+ }
735+ if _ , _ , err := runTestimTest (t , tc , "deploy-kots-application" ); err != nil {
736+ t .Fatalf ("fail to run testim test deploy-kots-application: %v" , err )
737+ }
738+
739+ // generate worker node join command.
740+ t .Logf ("%s: generating a new worker token command" , time .Now ().Format (time .RFC3339 ))
741+ stdout , stderr , err := runTestimTest (t , tc , "get-join-worker-command" )
742+ if err != nil {
743+ t .Fatalf ("fail to generate worker join token:\n stdout: %s\n stderr: %s" , stdout , stderr )
744+ }
745+ workerCommand , err := findJoinCommandInOutput (stdout )
746+ if err != nil {
747+ t .Fatalf ("fail to find the join command in the output: %v" , err )
748+ }
749+ t .Log ("worker join token command:" , workerCommand )
750+
751+ // join the worker node
752+ t .Logf ("%s: preparing embedded cluster airgap files on worker node" , time .Now ().Format (time .RFC3339 ))
753+ line = []string {"airgap-prepare.sh" }
754+ if _ , _ , err := RunCommandOnNode (t , tc , 1 , line ); err != nil {
755+ t .Fatalf ("fail to prepare airgap files on worker node: %v" , err )
756+ }
757+ t .Logf ("%s: joining worker node to the cluster" , time .Now ().Format (time .RFC3339 ))
758+ if _ , _ , err := RunCommandOnNode (t , tc , 1 , strings .Split (workerCommand , " " )); err != nil {
759+ t .Fatalf ("fail to join worker node to the cluster: %v" , err )
760+ }
761+ // remove the airgap bundle after joining
762+ line = []string {"rm" , "/tmp/release.airgap" }
763+ if _ , _ , err := RunCommandOnNode (t , tc , 1 , line ); err != nil {
764+ t .Fatalf ("fail to remove airgap bundle on worker node: %v" , err )
765+ }
766+
767+ // wait for the nodes to report as ready.
768+ t .Logf ("%s: all nodes joined, waiting for them to be ready" , time .Now ().Format (time .RFC3339 ))
769+ stdout , _ , err = RunCommandOnNode (t , tc , 0 , []string {"wait-for-ready-nodes.sh" , "2" })
770+ if err != nil {
771+ t .Fatalf ("fail to wait for ready nodes: %v" , err )
772+ }
773+ t .Log (stdout )
774+
775+ t .Logf ("%s: checking installation state after app deployment" , time .Now ().Format (time .RFC3339 ))
776+ line = []string {"check-airgap-installation-state.sh" , os .Getenv ("SHORT_SHA" )}
777+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
778+ t .Fatalf ("fail to check installation state: %v" , err )
779+ }
780+
781+ t .Logf ("%s: running airgap update" , time .Now ().Format (time .RFC3339 ))
782+ line = []string {"airgap-update.sh" }
783+ if _ , _ , err := RunCommandOnNode (t , tc , 0 , line ); err != nil {
784+ t .Fatalf ("fail to run airgap update: %v" , err )
648785 }
649786 // remove the airgap bundle after upgrade
650787 line = []string {"rm" , "/tmp/upgrade/release.airgap" }
0 commit comments