fix bug when using big yaml on the scanner

Author: tanjunchen

staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go

@@ -92,6 +92,10 @@ type YAMLDecoder struct {
 // the caller in framing the chunk.
 func NewDocumentDecoder(r io.ReadCloser) io.ReadCloser {
 	scanner := bufio.NewScanner(r)
+	// the size of initial allocation for buffer 4k
+	buf := make([]byte, 4*1024)
+	// the maximum size used to buffer a token 5M
+	scanner.Buffer(buf, 5*1024*1024)
 	scanner.Split(splitYAMLDocument)
 	return &YAMLDecoder{
 		r:       r,

staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go

@@ -54,6 +54,33 @@ stuff: 1
 	}
 }
 
+func TestBigYAML(t *testing.T) {
+	d := `
+stuff: 1
+`
+	maxLen := 5 * 1024 * 1024
+	bufferLen := 4 * 1024
+	//  maxLen 5 M
+	dd := strings.Repeat(d, 512*1024)
+	r := NewDocumentDecoder(ioutil.NopCloser(bytes.NewReader([]byte(dd[:maxLen-1]))))
+	b := make([]byte, bufferLen)
+	n, err := r.Read(b)
+	if err != io.ErrShortBuffer {
+		t.Fatalf("expected ErrShortBuffer: %d / %v", n, err)
+	}
+	b = make([]byte, maxLen)
+	n, err = r.Read(b)
+	if err != nil {
+		t.Fatalf("expected nil: %d / %v", n, err)
+	}
+	r = NewDocumentDecoder(ioutil.NopCloser(bytes.NewReader([]byte(dd))))
+	b = make([]byte, maxLen)
+	n, err = r.Read(b)
+	if err != bufio.ErrTooLong {
+		t.Fatalf("bufio.Scanner: token too long: %d / %v", n, err)
+	}
+}
+
 func TestYAMLDecoderCallsAfterErrShortBufferRestOfFrame(t *testing.T) {
 	d := `---
 stuff: 1