@@ -17,23 +17,12 @@ package crio
1717import (
1818 "encoding/json"
1919 "fmt"
20- "os"
21- "path"
22- "strings"
23- "time"
24-
25- "golang.org/x/crypto/ssh"
26- "golang.org/x/net/context"
2720
2821 "github.com/coreos/coreos-assembler/mantle/kola"
2922 "github.com/coreos/coreos-assembler/mantle/kola/cluster"
3023 "github.com/coreos/coreos-assembler/mantle/kola/register"
31- "github.com/coreos/coreos-assembler/mantle/kola/tests/util"
32- "github.com/coreos/coreos-assembler/mantle/lang/worker"
3324 "github.com/coreos/coreos-assembler/mantle/platform"
3425 "github.com/coreos/coreos-assembler/mantle/platform/conf"
35-
36- "github.com/google/uuid"
3726)
3827
3928// simplifiedCrioInfo represents the results from crio info
@@ -191,214 +180,18 @@ func init() {
191180 Run : crioBaseTests ,
192181 ClusterSize : 1 ,
193182 Name : `crio.base` ,
194- Description : "Verify cri-o basic funcions work, include storage driver is overlay, storage root is /varlib/containers/storage, cgroup driver is systemd, and cri-o containers have reliable networking " ,
183+ Description : "Verify cri-o basic funcions work, include storage driver is overlay, storage root is /varlib/containers/storage, cgroup driver is systemd" ,
195184 Distros : []string {"rhcos" },
196185 UserData : enableCrioIgn ,
197186 // crio pods require fetching a kubernetes pause image
198187 Tags : []string {"crio" , kola .NeedsInternetTag },
199188 RequiredTag : "openshift" ,
200189 })
201- register .RegisterTest (& register.Test {
202- Run : crioNetwork ,
203- ClusterSize : 2 ,
204- Name : "crio.network" ,
205- Description : "Verify crio containers can make network connections outside of the host." ,
206- Distros : []string {"rhcos" },
207- UserData : enableCrioIgn ,
208- // this test requires net connections outside the host
209- Tags : []string {"crio" , kola .NeedsInternetTag },
210- RequiredTag : "openshift" ,
211- // qemu machines cannot communicate between each other
212- ExcludePlatforms : []string {"qemu" },
213- })
214190}
215191
216192// crioBaseTests executes multiple tests under the "base" name
217193func crioBaseTests (c cluster.TestCluster ) {
218194 c .Run ("crio-info" , testCrioInfo )
219- c .Run ("pod-continues-during-service-restart" , crioPodContinuesDuringServiceRestart )
220- c .Run ("networks-reliably" , crioNetworksReliably )
221- }
222-
223- // generateCrioConfig generates a crio pod/container configuration
224- // based on the input name and arguments returning the path to the generated configs.
225- func generateCrioConfig (podName , imageName string , command []string ) (string , string , error ) {
226- fileContentsPod := fmt .Sprintf (crioPodTemplate , podName , uuid .New ().String (), imageName )
227-
228- tmpFilePod , err := os .CreateTemp ("" , podName + "Pod" )
229- if err != nil {
230- return "" , "" , err
231- }
232- defer tmpFilePod .Close ()
233- if _ , err = tmpFilePod .Write ([]byte (fileContentsPod )); err != nil {
234- return "" , "" , err
235- }
236- cmd := strings .Join (command , " " )
237- fileContentsContainer := fmt .Sprintf (crioContainerTemplate , imageName , imageName , cmd )
238-
239- tmpFileContainer , err := os .CreateTemp ("" , imageName + "Container" )
240- if err != nil {
241- return "" , "" , err
242- }
243- defer tmpFileContainer .Close ()
244- if _ , err = tmpFileContainer .Write ([]byte (fileContentsContainer )); err != nil {
245- return "" , "" , err
246- }
247-
248- return tmpFilePod .Name (), tmpFileContainer .Name (), nil
249- }
250-
251- // genContainer makes a container out of binaries on the host. This function uses podman to build.
252- // The first string returned by this function is the pod config to be used with crictl runp. The second
253- // string returned is the container config to be used with crictl create/exec. They will be dropped
254- // on to all machines in the cluster as ~/$STRING_RETURNED_FROM_FUNCTION. Note that the string returned
255- // here is just the name, not the full path on the cluster machine(s).
256- func genContainer (c cluster.TestCluster , m platform.Machine , podName , imageName string , binnames []string , shellCommands []string ) (string , string , error ) {
257- configPathPod , configPathContainer , err := generateCrioConfig (podName , imageName , shellCommands )
258- if err != nil {
259- return "" , "" , err
260- }
261- if err = cluster .DropFile (c .Machines (), configPathPod ); err != nil {
262- return "" , "" , err
263- }
264- if err = cluster .DropFile (c .Machines (), configPathContainer ); err != nil {
265- return "" , "" , err
266- }
267- // Create the crio image used for testing, only if it doesn't exist already
268- output := c .MustSSH (m , "sudo podman images -n --format '{{.Repository}}'" )
269- if ! strings .Contains (string (output ), "localhost/" + imageName ) {
270- util .GenPodmanScratchContainer (c , m , imageName , binnames )
271- }
272-
273- return path .Base (configPathPod ), path .Base (configPathContainer ), nil
274- }
275-
276- // crioNetwork ensures that crio containers can make network connections outside of the host
277- func crioNetwork (c cluster.TestCluster ) {
278- machines := c .Machines ()
279- src , dest := machines [0 ], machines [1 ]
280-
281- c .Log ("creating ncat containers" )
282-
283- // Since genContainer also generates crio pod/container configs,
284- // there will be a duplicate config file on each machine.
285- // Thus we only save one set for later use.
286- crioConfigPod , crioConfigContainer , err := genContainer (c , src , "ncat" , "ncat" , []string {"ncat" , "echo" }, []string {"ncat" })
287- if err != nil {
288- c .Fatal (err )
289- }
290- _ , _ , err = genContainer (c , dest , "ncat" , "ncat" , []string {"ncat" , "echo" }, []string {"ncat" })
291- if err != nil {
292- c .Fatal (err )
293- }
294-
295- listener := func (ctx context.Context ) error {
296- podID , err := c .SSHf (dest , "sudo crictl runp -T %s %s" , overrideCrioOperationTimeoutSeconds , crioConfigPod )
297- if err != nil {
298- return err
299- }
300-
301- containerID , err := c .SSHf (dest , "sudo crictl create -T %s --no-pull %s %s %s" ,
302- overrideCrioOperationTimeoutSeconds ,
303- podID , crioConfigContainer , crioConfigPod )
304- if err != nil {
305- return err
306- }
307-
308- // This command will block until a message is recieved
309- output , err := c .SSHf (dest , "sudo timeout 30 crictl exec %s echo 'HELLO FROM SERVER' | timeout 20 ncat --listen 0.0.0.0 9988 || echo 'LISTENER TIMEOUT'" , containerID )
310- if err != nil {
311- return err
312- }
313- if string (output ) != "HELLO FROM CLIENT" {
314- return fmt .Errorf ("unexpected result from listener: %s" , output )
315- }
316-
317- return nil
318- }
319-
320- talker := func (ctx context.Context ) error {
321- // Wait until listener is ready before trying anything
322- for {
323- _ , err := c .SSH (dest , "sudo ss -tulpn|grep 9988" )
324- if err == nil {
325- break // socket is ready
326- }
327-
328- exit , ok := err .(* ssh.ExitError )
329- if ! ok || exit .Waitmsg .ExitStatus () != 1 { // 1 is the expected exit of grep -q
330- return err
331- }
332-
333- select {
334- case <- ctx .Done ():
335- return fmt .Errorf ("timeout waiting for server" )
336- default :
337- time .Sleep (100 * time .Millisecond )
338- }
339- }
340- podID , err := c .SSHf (src , "sudo crictl runp -T %s %s" , overrideCrioOperationTimeoutSeconds , crioConfigPod )
341- if err != nil {
342- return err
343- }
344-
345- containerID , err := c .SSHf (src , "sudo crictl create -T %s --no-pull %s %s %s" ,
346- overrideCrioOperationTimeoutSeconds ,
347- podID , crioConfigContainer , crioConfigPod )
348- if err != nil {
349- return err
350- }
351-
352- output , err := c .SSHf (src , "sudo crictl exec %s echo 'HELLO FROM CLIENT' | ncat %s 9988" ,
353- containerID , dest .PrivateIP ())
354- if err != nil {
355- return err
356- }
357- if string (output ) != "HELLO FROM SERVER" {
358- return fmt .Errorf (`unexpected result from listener: "%s"` , output )
359- }
360-
361- return nil
362- }
363-
364- ctx , cancel := context .WithTimeout (context .Background (), time .Minute )
365- defer cancel ()
366-
367- if err := worker .Parallel (ctx , listener , talker ); err != nil {
368- c .Fatal (err )
369- }
370- }
371-
372- // crioNetworksReliably verifies that crio containers have a reliable network
373- func crioNetworksReliably (c cluster.TestCluster ) {
374- m := c .Machines ()[0 ]
375- hostIP := "127.0.0.1"
376-
377- // Here we generate 10 pods, each will run a container responsible for
378- // pinging to host
379- output := ""
380- for x := 1 ; x <= 10 ; x ++ {
381- // append int to name to avoid pod name collision
382- crioConfigPod , crioConfigContainer , err := genContainer (
383- c , m , fmt .Sprintf ("ping%d" , x ), "ping" , []string {"ping" },
384- []string {"ping" })
385- if err != nil {
386- c .Fatal (err )
387- }
388-
389- cmdCreatePod := fmt .Sprintf ("sudo crictl runp -T %s %s" , overrideCrioOperationTimeoutSeconds , crioConfigPod )
390- podID := c .MustSSH (m , cmdCreatePod )
391- containerID := c .MustSSH (m , fmt .Sprintf ("sudo crictl create -T %s --no-pull %s %s %s" ,
392- overrideCrioOperationTimeoutSeconds ,
393- podID , crioConfigContainer , crioConfigPod ))
394- output = output + string (c .MustSSH (m , fmt .Sprintf ("sudo crictl exec %s ping -i 0.2 %s -w 1 >/dev/null && echo PASS || echo FAIL" , containerID , hostIP )))
395- }
396-
397- numPass := strings .Count (string (output ), "PASS" )
398- if numPass != 10 {
399- c .Fatalf ("Expected 10 passes, but received %d passes with output: %s" , numPass , output )
400- }
401-
402195}
403196
404197// getCrioInfo parses and returns the information crio provides via socket
@@ -437,32 +230,3 @@ func testCrioInfo(c cluster.TestCluster) {
437230 }
438231
439232}
440-
441- // crioPodContinuesDuringServiceRestart verifies that a crio pod does not
442- // stop when the service is restarted
443- func crioPodContinuesDuringServiceRestart (c cluster.TestCluster ) {
444- m := c .Machines ()[0 ]
445-
446- crioConfigPod , crioConfigContainer , err := genContainer (
447- c , m , "restart-test" , "sleep" ,
448- []string {"bash" , "sleep" , "echo" }, []string {"bash" })
449- if err != nil {
450- c .Fatal (err )
451- }
452- cmdCreatePod := fmt .Sprintf ("sudo crictl runp -T %s %s" , overrideCrioOperationTimeoutSeconds , crioConfigPod )
453- podID := c .MustSSH (m , cmdCreatePod )
454- containerID := c .MustSSH (m , fmt .Sprintf ("sudo crictl create -T %s --no-pull %s %s %s" ,
455- overrideCrioOperationTimeoutSeconds ,
456- podID , crioConfigContainer , crioConfigPod ))
457-
458- cmd := fmt .Sprintf ("sudo crictl exec %s bash -c \" sleep 25 && echo PASS > /tmp/test/restart-test\" " , containerID )
459- c .RunCmdSync (m , cmd )
460- time .Sleep (3 * time .Second )
461- c .RunCmdSync (m , "sudo systemctl restart crio" )
462- time .Sleep (25 * time .Second )
463- output := strings .TrimSuffix (string (c .MustSSH (m , "cat /tmp/test/restart-test" )), "\n " )
464-
465- if output != "PASS" {
466- c .Fatalf ("Pod did not continue during service restart. Output=%s, Command=%s" , output , cmd )
467- }
468- }
0 commit comments