Skip to content

Commit 7151131

Browse files
authored
Merge pull request kubernetes#73032 from liggitt/kubectl-warning
surface server-side warnings in client-go / kubectl
2 parents 2402bfd + df6608d commit 7151131

File tree

37 files changed

+1836
-36
lines changed

37 files changed

+1836
-36
lines changed

cmd/kube-apiserver/app/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ go_library(
5555
"//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library",
5656
"//staging/src/k8s.io/client-go/informers:go_default_library",
5757
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
58+
"//staging/src/k8s.io/client-go/rest:go_default_library",
5859
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
5960
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
6061
"//staging/src/k8s.io/cloud-provider:go_default_library",

cmd/kube-apiserver/app/server.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ import (
5353
"k8s.io/apiserver/pkg/util/webhook"
5454
clientgoinformers "k8s.io/client-go/informers"
5555
clientgoclientset "k8s.io/client-go/kubernetes"
56+
"k8s.io/client-go/rest"
5657
"k8s.io/client-go/util/keyutil"
5758
cloudprovider "k8s.io/cloud-provider"
5859
cliflag "k8s.io/component-base/cli/flag"
@@ -99,6 +100,12 @@ cluster's shared state through which all other components interact.`,
99100

100101
// stop printing usage when the command errors
101102
SilenceUsage: true,
103+
PersistentPreRunE: func(*cobra.Command, []string) error {
104+
// silence client-go warnings.
105+
// kube-apiserver loopback clients should not log self-issued warnings.
106+
rest.SetDefaultWarningHandler(rest.NoWarnings{})
107+
return nil
108+
},
102109
RunE: func(cmd *cobra.Command, args []string) error {
103110
verflag.PrintAndExitIfRequested()
104111
cliflag.PrintFlags(cmd.Flags())

cmd/kube-controller-manager/app/controllermanager.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,13 @@ state of the cluster through the apiserver and makes changes attempting to move
104104
current state towards the desired state. Examples of controllers that ship with
105105
Kubernetes today are the replication controller, endpoints controller, namespace
106106
controller, and serviceaccounts controller.`,
107+
PersistentPreRunE: func(*cobra.Command, []string) error {
108+
// silence client-go warnings.
109+
// kube-controller-manager generically watches APIs (including deprecated ones),
110+
// and CI ensures it works properly against matching kube-apiserver versions.
111+
restclient.SetDefaultWarningHandler(restclient.NoWarnings{})
112+
return nil
113+
},
107114
Run: func(cmd *cobra.Command, args []string) {
108115
verflag.PrintAndExitIfRequested()
109116
cliflag.PrintFlags(cmd.Flags())

pkg/features/kube_features.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
679679
genericfeatures.DryRun: {Default: true, PreRelease: featuregate.GA},
680680
genericfeatures.ServerSideApply: {Default: true, PreRelease: featuregate.Beta},
681681
genericfeatures.APIPriorityAndFairness: {Default: false, PreRelease: featuregate.Alpha},
682+
genericfeatures.WarningHeaders: {Default: true, PreRelease: featuregate.Beta},
682683

683684
// features that enable backwards compatibility but are scheduled to be removed
684685
// ...

pkg/kubeapiserver/server/insecure_handler.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ func BuildInsecureHandlerChain(apiHandler http.Handler, c *server.Config) http.H
4141
handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc, c.RequestTimeout)
4242
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
4343
handler = genericapifilters.WithRequestInfo(handler, server.NewRequestInfoResolver(c))
44+
handler = genericapifilters.WithWarningRecorder(handler)
4445
handler = genericapifilters.WithCacheControl(handler)
4546
handler = genericfilters.WithPanicRecovery(handler)
4647

pkg/kubectl/cmd/BUILD

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ go_library(
1313
"//pkg/kubectl/cmd/convert:go_default_library",
1414
"//pkg/kubectl/cmd/cp:go_default_library",
1515
"//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library",
16+
"//staging/src/k8s.io/client-go/rest:go_default_library",
1617
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
1718
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
1819
"//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library",
@@ -55,6 +56,7 @@ go_library(
5556
"//staging/src/k8s.io/kubectl/pkg/cmd/wait:go_default_library",
5657
"//staging/src/k8s.io/kubectl/pkg/util/i18n:go_default_library",
5758
"//staging/src/k8s.io/kubectl/pkg/util/templates:go_default_library",
59+
"//staging/src/k8s.io/kubectl/pkg/util/term:go_default_library",
5860
"//vendor/github.com/spf13/cobra:go_default_library",
5961
"//vendor/github.com/spf13/pflag:go_default_library",
6062
],

pkg/kubectl/cmd/cmd.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
"github.com/spf13/cobra"
3030

31+
"k8s.io/client-go/rest"
3132
"k8s.io/client-go/tools/clientcmd"
3233
cliflag "k8s.io/component-base/cli/flag"
3334
cmdpkg "k8s.io/kubectl/pkg/cmd"
@@ -69,6 +70,7 @@ import (
6970
"k8s.io/kubectl/pkg/cmd/wait"
7071
"k8s.io/kubectl/pkg/util/i18n"
7172
"k8s.io/kubectl/pkg/util/templates"
73+
"k8s.io/kubectl/pkg/util/term"
7274
"k8s.io/kubernetes/pkg/kubectl/cmd/auth"
7375
"k8s.io/kubernetes/pkg/kubectl/cmd/convert"
7476
"k8s.io/kubernetes/pkg/kubectl/cmd/cp"
@@ -428,6 +430,9 @@ func HandlePluginCommand(pluginHandler PluginHandler, cmdArgs []string) error {
428430

429431
// NewKubectlCommand creates the `kubectl` command and its nested children.
430432
func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command {
433+
warningHandler := rest.NewWarningWriter(err, rest.WarningWriterOptions{Deduplicate: true, Color: term.AllowsColorOutput(err)})
434+
warningsAsErrors := false
435+
431436
// Parent command to which all subcommands are added.
432437
cmds := &cobra.Command{
433438
Use: "kubectl",
@@ -441,10 +446,25 @@ func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command {
441446
// Hook before and after Run initialize and write profiles to disk,
442447
// respectively.
443448
PersistentPreRunE: func(*cobra.Command, []string) error {
449+
rest.SetDefaultWarningHandler(warningHandler)
444450
return initProfiling()
445451
},
446452
PersistentPostRunE: func(*cobra.Command, []string) error {
447-
return flushProfiling()
453+
if err := flushProfiling(); err != nil {
454+
return err
455+
}
456+
if warningsAsErrors {
457+
count := warningHandler.WarningCount()
458+
switch count {
459+
case 0:
460+
// no warnings
461+
case 1:
462+
return fmt.Errorf("%d warning received", count)
463+
default:
464+
return fmt.Errorf("%d warnings received", count)
465+
}
466+
}
467+
return nil
448468
},
449469
BashCompletionFunction: bashCompletionFunc,
450470
}
@@ -458,6 +478,8 @@ func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command {
458478

459479
addProfilingFlags(flags)
460480

481+
flags.BoolVar(&warningsAsErrors, "warnings-as-errors", warningsAsErrors, "Treat warnings received from the server as errors and exit with a non-zero exit code")
482+
461483
kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
462484
kubeConfigFlags.AddFlags(flags)
463485
matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags)

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
360360
}
361361

362362
if handlerFunc != nil {
363-
handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, handlerFunc)
363+
handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, false, "", handlerFunc)
364364
handler := genericfilters.WithWaitGroup(handlerFunc, longRunningFilter, crdInfo.waitGroup)
365365
handler.ServeHTTP(w, req)
366366
return

staging/src/k8s.io/apimachinery/pkg/util/net/http.go

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,20 @@ import (
2121
"bytes"
2222
"context"
2323
"crypto/tls"
24+
"errors"
2425
"fmt"
2526
"io"
27+
"mime"
2628
"net"
2729
"net/http"
2830
"net/url"
2931
"os"
3032
"path"
33+
"regexp"
3134
"strconv"
3235
"strings"
36+
"unicode"
37+
"unicode/utf8"
3338

3439
"golang.org/x/net/http2"
3540
"k8s.io/klog/v2"
@@ -482,3 +487,232 @@ func CloneHeader(in http.Header) http.Header {
482487
}
483488
return out
484489
}
490+
491+
// WarningHeader contains a single RFC2616 14.46 warnings header
492+
type WarningHeader struct {
493+
// Codeindicates the type of warning. 299 is a miscellaneous persistent warning
494+
Code int
495+
// Agent contains the name or pseudonym of the server adding the Warning header.
496+
// A single "-" is recommended when agent is unknown.
497+
Agent string
498+
// Warning text
499+
Text string
500+
}
501+
502+
// ParseWarningHeaders extract RFC2616 14.46 warnings headers from the specified set of header values.
503+
// Multiple comma-separated warnings per header are supported.
504+
// If errors are encountered on a header, the remainder of that header are skipped and subsequent headers are parsed.
505+
// Returns successfully parsed warnings and any errors encountered.
506+
func ParseWarningHeaders(headers []string) ([]WarningHeader, []error) {
507+
var (
508+
results []WarningHeader
509+
errs []error
510+
)
511+
for _, header := range headers {
512+
for len(header) > 0 {
513+
result, remainder, err := ParseWarningHeader(header)
514+
if err != nil {
515+
errs = append(errs, err)
516+
break
517+
}
518+
results = append(results, result)
519+
header = remainder
520+
}
521+
}
522+
return results, errs
523+
}
524+
525+
var (
526+
codeMatcher = regexp.MustCompile(`^[0-9]{3}$`)
527+
wordDecoder = &mime.WordDecoder{}
528+
)
529+
530+
// ParseWarningHeader extracts one RFC2616 14.46 warning from the specified header,
531+
// returning an error if the header does not contain a correctly formatted warning.
532+
// Any remaining content in the header is returned.
533+
func ParseWarningHeader(header string) (result WarningHeader, remainder string, err error) {
534+
// https://tools.ietf.org/html/rfc2616#section-14.46
535+
// updated by
536+
// https://tools.ietf.org/html/rfc7234#section-5.5
537+
// https://tools.ietf.org/html/rfc7234#appendix-A
538+
// Some requirements regarding production and processing of the Warning
539+
// header fields have been relaxed, as it is not widely implemented.
540+
// Furthermore, the Warning header field no longer uses RFC 2047
541+
// encoding, nor does it allow multiple languages, as these aspects were
542+
// not implemented.
543+
//
544+
// Format is one of:
545+
// warn-code warn-agent "warn-text"
546+
// warn-code warn-agent "warn-text" "warn-date"
547+
//
548+
// warn-code is a three digit number
549+
// warn-agent is unquoted and contains no spaces
550+
// warn-text is quoted with backslash escaping (RFC2047-encoded according to RFC2616, not encoded according to RFC7234)
551+
// warn-date is optional, quoted, and in HTTP-date format (no embedded or escaped quotes)
552+
//
553+
// additional warnings can optionally be included in the same header by comma-separating them:
554+
// warn-code warn-agent "warn-text" "warn-date"[, warn-code warn-agent "warn-text" "warn-date", ...]
555+
556+
// tolerate leading whitespace
557+
header = strings.TrimSpace(header)
558+
559+
parts := strings.SplitN(header, " ", 3)
560+
if len(parts) != 3 {
561+
return WarningHeader{}, "", errors.New("invalid warning header: fewer than 3 segments")
562+
}
563+
code, agent, textDateRemainder := parts[0], parts[1], parts[2]
564+
565+
// verify code format
566+
if !codeMatcher.Match([]byte(code)) {
567+
return WarningHeader{}, "", errors.New("invalid warning header: code segment is not 3 digits between 100-299")
568+
}
569+
codeInt, _ := strconv.ParseInt(code, 10, 64)
570+
571+
// verify agent presence
572+
if len(agent) == 0 {
573+
return WarningHeader{}, "", errors.New("invalid warning header: empty agent segment")
574+
}
575+
if !utf8.ValidString(agent) || hasAnyRunes(agent, unicode.IsControl) {
576+
return WarningHeader{}, "", errors.New("invalid warning header: invalid agent")
577+
}
578+
579+
// verify textDateRemainder presence
580+
if len(textDateRemainder) == 0 {
581+
return WarningHeader{}, "", errors.New("invalid warning header: empty text segment")
582+
}
583+
584+
// extract text
585+
text, dateAndRemainder, err := parseQuotedString(textDateRemainder)
586+
if err != nil {
587+
return WarningHeader{}, "", fmt.Errorf("invalid warning header: %v", err)
588+
}
589+
// tolerate RFC2047-encoded text from warnings produced according to RFC2616
590+
if decodedText, err := wordDecoder.DecodeHeader(text); err == nil {
591+
text = decodedText
592+
}
593+
if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) {
594+
return WarningHeader{}, "", errors.New("invalid warning header: invalid text")
595+
}
596+
result = WarningHeader{Code: int(codeInt), Agent: agent, Text: text}
597+
598+
if len(dateAndRemainder) > 0 {
599+
if dateAndRemainder[0] == '"' {
600+
// consume date
601+
foundEndQuote := false
602+
for i := 1; i < len(dateAndRemainder); i++ {
603+
if dateAndRemainder[i] == '"' {
604+
foundEndQuote = true
605+
remainder = strings.TrimSpace(dateAndRemainder[i+1:])
606+
break
607+
}
608+
}
609+
if !foundEndQuote {
610+
return WarningHeader{}, "", errors.New("invalid warning header: unterminated date segment")
611+
}
612+
} else {
613+
remainder = dateAndRemainder
614+
}
615+
}
616+
if len(remainder) > 0 {
617+
if remainder[0] == ',' {
618+
// consume comma if present
619+
remainder = strings.TrimSpace(remainder[1:])
620+
} else {
621+
return WarningHeader{}, "", errors.New("invalid warning header: unexpected token after warn-date")
622+
}
623+
}
624+
625+
return result, remainder, nil
626+
}
627+
628+
func parseQuotedString(quotedString string) (string, string, error) {
629+
if len(quotedString) == 0 {
630+
return "", "", errors.New("invalid quoted string: 0-length")
631+
}
632+
633+
if quotedString[0] != '"' {
634+
return "", "", errors.New("invalid quoted string: missing initial quote")
635+
}
636+
637+
quotedString = quotedString[1:]
638+
var remainder string
639+
escaping := false
640+
closedQuote := false
641+
result := &bytes.Buffer{}
642+
loop:
643+
for i := 0; i < len(quotedString); i++ {
644+
b := quotedString[i]
645+
switch b {
646+
case '"':
647+
if escaping {
648+
result.WriteByte(b)
649+
escaping = false
650+
} else {
651+
closedQuote = true
652+
remainder = strings.TrimSpace(quotedString[i+1:])
653+
break loop
654+
}
655+
case '\\':
656+
if escaping {
657+
result.WriteByte(b)
658+
escaping = false
659+
} else {
660+
escaping = true
661+
}
662+
default:
663+
result.WriteByte(b)
664+
escaping = false
665+
}
666+
}
667+
668+
if !closedQuote {
669+
return "", "", errors.New("invalid quoted string: missing closing quote")
670+
}
671+
return result.String(), remainder, nil
672+
}
673+
674+
func NewWarningHeader(code int, agent, text string) (string, error) {
675+
if code < 0 || code > 999 {
676+
return "", errors.New("code must be between 0 and 999")
677+
}
678+
if len(agent) == 0 {
679+
agent = "-"
680+
} else if !utf8.ValidString(agent) || strings.ContainsAny(agent, `\"`) || hasAnyRunes(agent, unicode.IsSpace, unicode.IsControl) {
681+
return "", errors.New("agent must be valid UTF-8 and must not contain spaces, quotes, backslashes, or control characters")
682+
}
683+
if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) {
684+
return "", errors.New("text must be valid UTF-8 and must not contain control characters")
685+
}
686+
return fmt.Sprintf("%03d %s %s", code, agent, makeQuotedString(text)), nil
687+
}
688+
689+
func hasAnyRunes(s string, runeCheckers ...func(rune) bool) bool {
690+
for _, r := range s {
691+
for _, checker := range runeCheckers {
692+
if checker(r) {
693+
return true
694+
}
695+
}
696+
}
697+
return false
698+
}
699+
700+
func makeQuotedString(s string) string {
701+
result := &bytes.Buffer{}
702+
// opening quote
703+
result.WriteRune('"')
704+
for _, c := range s {
705+
switch c {
706+
case '"', '\\':
707+
// escape " and \
708+
result.WriteRune('\\')
709+
result.WriteRune(c)
710+
default:
711+
// write everything else as-is
712+
result.WriteRune(c)
713+
}
714+
}
715+
// closing quote
716+
result.WriteRune('"')
717+
return result.String()
718+
}

0 commit comments

Comments
 (0)