gopkg.in/yaml.v3/readerc.go

// 
// Copyright (c) 2011-2019 Canonical Ltd
// Copyright (c) 2006-2010 Kirill Simonov
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package yaml

import (
        "io"
)

// Set the reader error and return 0.
func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
        parser.error = yaml_READER_ERROR
        parser.problem = problem
        parser.problem_offset = offset
        parser.problem_value = value
        return false
}

// Byte order marks.
const (
        bom_UTF8    = "\xef\xbb\xbf"
        bom_UTF16LE = "\xff\xfe"
        bom_UTF16BE = "\xfe\xff"
)

// Determine the input stream encoding by checking the BOM symbol. If no BOM is
// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
        // Ensure that we had enough bytes in the raw buffer.
        for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
                if !yaml_parser_update_raw_buffer(parser) {
                        return false
                }
        }

        // Determine the encoding.
        buf := parser.raw_buffer
        pos := parser.raw_buffer_pos
        avail := len(buf) - pos
        if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
                parser.encoding = yaml_UTF16LE_ENCODING
                parser.raw_buffer_pos += 2
                parser.offset += 2
        } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
                parser.encoding = yaml_UTF16BE_ENCODING
                parser.raw_buffer_pos += 2
                parser.offset += 2
        } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
                parser.encoding = yaml_UTF8_ENCODING
                parser.raw_buffer_pos += 3
                parser.offset += 3
        } else {
                parser.encoding = yaml_UTF8_ENCODING
        }
        return true
}

// Update the raw buffer.
func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
        size_read := 0

        // Return if the raw buffer is full.
        if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
                return true
        }

        // Return on EOF.
        if parser.eof {
                return true
        }

        // Move the remaining bytes in the raw buffer to the beginning.
        if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
                copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
        }
        parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
        parser.raw_buffer_pos = 0

        // Call the read handler to fill the buffer.
        size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
        parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
        if err == io.EOF {
                parser.eof = true
        } else if err != nil {
                return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
        }
        return true
}

// Ensure that the buffer contains at least `length` characters.
// Return true on success, false on failure.
//
// The length is supposed to be significantly less that the buffer size.
func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
        if parser.read_handler == nil {
                panic("read handler must be set")
        }

        // [Go] This function was changed to guarantee the requested length size at EOF.
        // The fact we need to do this is pretty awful, but the description above implies
        // for that to be the case, and there are tests

        // If the EOF flag is set and the raw buffer is empty, do nothing.
        if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
                // [Go] ACTUALLY! Read the documentation of this function above.
                // This is just broken. To return true, we need to have the
                // given length in the buffer. Not doing that means every single
                // check that calls this function to make sure the buffer has a
                // given length is Go) panicking; or C) accessing invalid memory.
                //return true
        }

        // Return if the buffer contains enough characters.
        if parser.unread >= length {
                return true
        }

        // Determine the input encoding if it is not known yet.
        if parser.encoding == yaml_ANY_ENCODING {
                if !yaml_parser_determine_encoding(parser) {
                        return false
                }
        }

        // Move the unread characters to the beginning of the buffer.
        buffer_len := len(parser.buffer)
        if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
                copy(parser.buffer, parser.buffer[parser.buffer_pos:])
                buffer_len -= parser.buffer_pos
                parser.buffer_pos = 0
        } else if parser.buffer_pos == buffer_len {
                buffer_len = 0
                parser.buffer_pos = 0
        }

        // Open the whole buffer for writing, and cut it before returning.
        parser.buffer = parser.buffer[:cap(parser.buffer)]

        // Fill the buffer until it has enough characters.
        first := true
        for parser.unread < length {

                // Fill the raw buffer if necessary.
                if !first || parser.raw_buffer_pos == len(parser.raw_buffer) {
                        if !yaml_parser_update_raw_buffer(parser) {
                                parser.buffer = parser.buffer[:buffer_len]
                                return false
                        }
                }
                first = false

                // Decode the raw buffer.
        inner:
                for parser.raw_buffer_pos != len(parser.raw_buffer) {
                        var value rune
                        var width int

                        raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos

                        // Decode the next character.
                        switch parser.encoding {
                        case yaml_UTF8_ENCODING:
                                // Decode a UTF-8 character.  Check RFC 3629
                                // (http://www.ietf.org/rfc/rfc3629.txt) for more details.
                                //
                                // The following table (taken from the RFC) is used for
                                // decoding.
                                //
                                //    Char. number range |        UTF-8 octet sequence
                                //      (hexadecimal)    |              (binary)
                                //   --------------------+------------------------------------
                                //   0000 0000-0000 007F | 0xxxxxxx
                                //   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
                                //   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
                                //   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                                //
                                // Additionally, the characters in the range 0xD800-0xDFFF
                                // are prohibited as they are reserved for use with UTF-16
                                // surrogate pairs.

                                // Determine the length of the UTF-8 sequence.
                                octet := parser.raw_buffer[parser.raw_buffer_pos]
                                switch {
                                case octet&0x80 == 0x00:
                                        width = 1
                                case octet&0xE0 == 0xC0:
                                        width = 2
                                case octet&0xF0 == 0xE0:
                                        width = 3
                                case octet&0xF8 == 0xF0:
                                        width = 4
                                default:
                                        // The leading octet is invalid.
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid leading UTF-8 octet",
                                                parser.offset, int(octet))
                                }

                                // Check if the raw buffer contains an incomplete character.
                                if width > raw_unread {
                                        if parser.eof {
                                                return yaml_parser_set_reader_error(parser,
                                                        "incomplete UTF-8 octet sequence",
                                                        parser.offset, -1)
                                        }
                                        break inner
                                }

                                // Decode the leading octet.
                                switch {
                                case octet&0x80 == 0x00:
                                        value = rune(octet & 0x7F)
                                case octet&0xE0 == 0xC0:
                                        value = rune(octet & 0x1F)
                                case octet&0xF0 == 0xE0:
                                        value = rune(octet & 0x0F)
                                case octet&0xF8 == 0xF0:
                                        value = rune(octet & 0x07)
                                default:
                                        value = 0
                                }

                                // Check and decode the trailing octets.
                                for k := 1; k < width; k++ {
                                        octet = parser.raw_buffer[parser.raw_buffer_pos+k]

                                        // Check if the octet is valid.
                                        if (octet & 0xC0) != 0x80 {
                                                return yaml_parser_set_reader_error(parser,
                                                        "invalid trailing UTF-8 octet",
                                                        parser.offset+k, int(octet))
                                        }

                                        // Decode the octet.
                                        value = (value << 6) + rune(octet&0x3F)
                                }

                                // Check the length of the sequence against the value.
                                switch {
                                case width == 1:
                                case width == 2 && value >= 0x80:
                                case width == 3 && value >= 0x800:
                                case width == 4 && value >= 0x10000:
                                default:
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid length of a UTF-8 sequence",
                                                parser.offset, -1)
                                }

                                // Check the range of the value.
                                if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF {
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid Unicode character",
                                                parser.offset, int(value))
                                }

                        case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
                                var low, high int
                                if parser.encoding == yaml_UTF16LE_ENCODING {
                                        low, high = 0, 1
                                } else {
                                        low, high = 1, 0
                                }

                                // The UTF-16 encoding is not as simple as one might
                                // naively think.  Check RFC 2781
                                // (http://www.ietf.org/rfc/rfc2781.txt).
                                //
                                // Normally, two subsequent bytes describe a Unicode
                                // character.  However a special technique (called a
                                // surrogate pair) is used for specifying character
                                // values larger than 0xFFFF.
                                //
                                // A surrogate pair consists of two pseudo-characters:
                                //      high surrogate area (0xD800-0xDBFF)
                                //      low surrogate area (0xDC00-0xDFFF)
                                //
                                // The following formulas are used for decoding
                                // and encoding characters using surrogate pairs:
                                //
                                //  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
                                //  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
                                //  W1 = 110110yyyyyyyyyy
                                //  W2 = 110111xxxxxxxxxx
                                //
                                // where U is the character value, W1 is the high surrogate
                                // area, W2 is the low surrogate area.

                                // Check for incomplete UTF-16 character.
                                if raw_unread < 2 {
                                        if parser.eof {
                                                return yaml_parser_set_reader_error(parser,
                                                        "incomplete UTF-16 character",
                                                        parser.offset, -1)
                                        }
                                        break inner
                                }

                                // Get the character.
                                value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
                                        (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)

                                // Check for unexpected low surrogate area.
                                if value&0xFC00 == 0xDC00 {
                                        return yaml_parser_set_reader_error(parser,
                                                "unexpected low surrogate area",
                                                parser.offset, int(value))
                                }

                                // Check for a high surrogate area.
                                if value&0xFC00 == 0xD800 {
                                        width = 4

                                        // Check for incomplete surrogate pair.
                                        if raw_unread < 4 {
                                                if parser.eof {
                                                        return yaml_parser_set_reader_error(parser,
                                                                "incomplete UTF-16 surrogate pair",
                                                                parser.offset, -1)
                                                }
                                                break inner
                                        }

                                        // Get the next character.
                                        value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
                                                (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)

                                        // Check for a low surrogate area.
                                        if value2&0xFC00 != 0xDC00 {
                                                return yaml_parser_set_reader_error(parser,
                                                        "expected low surrogate area",
                                                        parser.offset+2, int(value2))
                                        }

                                        // Generate the value of the surrogate pair.
                                        value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
                                } else {
                                        width = 2
                                }

                        default:
                                panic("impossible")
                        }

                        // Check if the character is in the allowed range:
                        //      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
                        //      | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD]    (16 bit)
                        //      | [#x10000-#x10FFFF]                        (32 bit)
                        switch {
                        case value == 0x09:
                        case value == 0x0A:
                        case value == 0x0D:
                        case value >= 0x20 && value <= 0x7E:
                        case value == 0x85:
                        case value >= 0xA0 && value <= 0xD7FF:
                        case value >= 0xE000 && value <= 0xFFFD:
                        case value >= 0x10000 && value <= 0x10FFFF:
                        default:
                                return yaml_parser_set_reader_error(parser,
                                        "control characters are not allowed",
                                        parser.offset, int(value))
                        }

                        // Move the raw pointers.
                        parser.raw_buffer_pos += width
                        parser.offset += width

                        // Finally put the character into the buffer.
                        if value <= 0x7F {
                                // 0000 0000-0000 007F . 0xxxxxxx
                                parser.buffer[buffer_len+0] = byte(value)
                                buffer_len += 1
                        } else if value <= 0x7FF {
                                // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
                                parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
                                buffer_len += 2
                        } else if value <= 0xFFFF {
                                // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
                                parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
                                parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
                                buffer_len += 3
                        } else {
                                // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
                                parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
                                parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
                                parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
                                buffer_len += 4
                        }

                        parser.unread++
                }

                // On EOF, put NUL into the buffer and return.
                if parser.eof {
                        parser.buffer[buffer_len] = 0
                        buffer_len++
                        parser.unread++
                        break
                }
        }
        // [Go] Read the documentation of this function above. To return true,
        // we need to have the given length in the buffer. Not doing that means
        // every single check that calls this function to make sure the buffer
        // has a given length is Go) panicking; or C) accessing invalid memory.
        // This happens here due to the EOF above breaking early.
        for buffer_len < length {
                parser.buffer[buffer_len] = 0
                buffer_len++
        }
        parser.buffer = parser.buffer[:buffer_len]
        return true
}
Revision:	1.1
Committed:	Mon Sep 30 00:42:06 2024 UTC (6 weeks, 4 days ago) by yakumo_izuru
Branch:	MAIN
CVS Tags:	HEAD
Error occurred while calculating annotation data.
Log Message:	Mirrored from https://git.chaotic.ninja/git/yakumo_izuru/aya
#	Content
1	//
2	// Copyright (c) 2011-2019 Canonical Ltd
3	// Copyright (c) 2006-2010 Kirill Simonov
4	//
5	// Permission is hereby granted, free of charge, to any person obtaining a copy of
6	// this software and associated documentation files (the "Software"), to deal in
7	// the Software without restriction, including without limitation the rights to
8	// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
9	// of the Software, and to permit persons to whom the Software is furnished to do
10	// so, subject to the following conditions:
11	//
12	// The above copyright notice and this permission notice shall be included in all
13	// copies or substantial portions of the Software.
14	//
15	// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18	// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20	// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21	// SOFTWARE.
22
23	package yaml
24
25	import (
26	"io"
27	)
28
29	// Set the reader error and return 0.
30	func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
31	parser.error = yaml_READER_ERROR
32	parser.problem = problem
33	parser.problem_offset = offset
34	parser.problem_value = value
35	return false
36	}
37
38	// Byte order marks.
39	const (
40	bom_UTF8 = "\xef\xbb\xbf"
41	bom_UTF16LE = "\xff\xfe"
42	bom_UTF16BE = "\xfe\xff"
43	)
44
45	// Determine the input stream encoding by checking the BOM symbol. If no BOM is
46	// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
47	func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
48	// Ensure that we had enough bytes in the raw buffer.
49	for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
50	if !yaml_parser_update_raw_buffer(parser) {
51	return false
52	}
53	}
54
55	// Determine the encoding.
56	buf := parser.raw_buffer
57	pos := parser.raw_buffer_pos
58	avail := len(buf) - pos
59	if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
60	parser.encoding = yaml_UTF16LE_ENCODING
61	parser.raw_buffer_pos += 2
62	parser.offset += 2
63	} else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
64	parser.encoding = yaml_UTF16BE_ENCODING
65	parser.raw_buffer_pos += 2
66	parser.offset += 2
67	} else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
68	parser.encoding = yaml_UTF8_ENCODING
69	parser.raw_buffer_pos += 3
70	parser.offset += 3
71	} else {
72	parser.encoding = yaml_UTF8_ENCODING
73	}
74	return true
75	}
76
77	// Update the raw buffer.
78	func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
79	size_read := 0
80
81	// Return if the raw buffer is full.
82	if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
83	return true
84	}
85
86	// Return on EOF.
87	if parser.eof {
88	return true
89	}
90
91	// Move the remaining bytes in the raw buffer to the beginning.
92	if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
93	copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
94	}
95	parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
96	parser.raw_buffer_pos = 0
97
98	// Call the read handler to fill the buffer.
99	size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
100	parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
101	if err == io.EOF {
102	parser.eof = true
103	} else if err != nil {
104	return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
105	}
106	return true
107	}
108
109	// Ensure that the buffer contains at least `length` characters.
110	// Return true on success, false on failure.
111	//
112	// The length is supposed to be significantly less that the buffer size.
113	func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
114	if parser.read_handler == nil {
115	panic("read handler must be set")
116	}
117
118	// [Go] This function was changed to guarantee the requested length size at EOF.
119	// The fact we need to do this is pretty awful, but the description above implies
120	// for that to be the case, and there are tests
121
122	// If the EOF flag is set and the raw buffer is empty, do nothing.
123	if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
124	// [Go] ACTUALLY! Read the documentation of this function above.
125	// This is just broken. To return true, we need to have the
126	// given length in the buffer. Not doing that means every single
127	// check that calls this function to make sure the buffer has a
128	// given length is Go) panicking; or C) accessing invalid memory.
129	//return true
130	}
131
132	// Return if the buffer contains enough characters.
133	if parser.unread >= length {
134	return true
135	}
136
137	// Determine the input encoding if it is not known yet.
138	if parser.encoding == yaml_ANY_ENCODING {
139	if !yaml_parser_determine_encoding(parser) {
140	return false
141	}
142	}
143
144	// Move the unread characters to the beginning of the buffer.
145	buffer_len := len(parser.buffer)
146	if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
147	copy(parser.buffer, parser.buffer[parser.buffer_pos:])
148	buffer_len -= parser.buffer_pos
149	parser.buffer_pos = 0
150	} else if parser.buffer_pos == buffer_len {
151	buffer_len = 0
152	parser.buffer_pos = 0
153	}
154
155	// Open the whole buffer for writing, and cut it before returning.
156	parser.buffer = parser.buffer[:cap(parser.buffer)]
157
158	// Fill the buffer until it has enough characters.
159	first := true
160	for parser.unread < length {
161
162	// Fill the raw buffer if necessary.
163	if !first \|\| parser.raw_buffer_pos == len(parser.raw_buffer) {
164	if !yaml_parser_update_raw_buffer(parser) {
165	parser.buffer = parser.buffer[:buffer_len]
166	return false
167	}
168	}
169	first = false
170
171	// Decode the raw buffer.
172	inner:
173	for parser.raw_buffer_pos != len(parser.raw_buffer) {
174	var value rune
175	var width int
176
177	raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos
178
179	// Decode the next character.
180	switch parser.encoding {
181	case yaml_UTF8_ENCODING:
182	// Decode a UTF-8 character. Check RFC 3629
183	// (http://www.ietf.org/rfc/rfc3629.txt) for more details.
184	//
185	// The following table (taken from the RFC) is used for
186	// decoding.
187	//
188	// Char. number range \| UTF-8 octet sequence
189	// (hexadecimal) \| (binary)
190	// --------------------+------------------------------------
191	// 0000 0000-0000 007F \| 0xxxxxxx
192	// 0000 0080-0000 07FF \| 110xxxxx 10xxxxxx
193	// 0000 0800-0000 FFFF \| 1110xxxx 10xxxxxx 10xxxxxx
194	// 0001 0000-0010 FFFF \| 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
195	//
196	// Additionally, the characters in the range 0xD800-0xDFFF
197	// are prohibited as they are reserved for use with UTF-16
198	// surrogate pairs.
199
200	// Determine the length of the UTF-8 sequence.
201	octet := parser.raw_buffer[parser.raw_buffer_pos]
202	switch {
203	case octet&0x80 == 0x00:
204	width = 1
205	case octet&0xE0 == 0xC0:
206	width = 2
207	case octet&0xF0 == 0xE0:
208	width = 3
209	case octet&0xF8 == 0xF0:
210	width = 4
211	default:
212	// The leading octet is invalid.
213	return yaml_parser_set_reader_error(parser,
214	"invalid leading UTF-8 octet",
215	parser.offset, int(octet))
216	}
217
218	// Check if the raw buffer contains an incomplete character.
219	if width > raw_unread {
220	if parser.eof {
221	return yaml_parser_set_reader_error(parser,
222	"incomplete UTF-8 octet sequence",
223	parser.offset, -1)
224	}
225	break inner
226	}
227
228	// Decode the leading octet.
229	switch {
230	case octet&0x80 == 0x00:
231	value = rune(octet & 0x7F)
232	case octet&0xE0 == 0xC0:
233	value = rune(octet & 0x1F)
234	case octet&0xF0 == 0xE0:
235	value = rune(octet & 0x0F)
236	case octet&0xF8 == 0xF0:
237	value = rune(octet & 0x07)
238	default:
239	value = 0
240	}
241
242	// Check and decode the trailing octets.
243	for k := 1; k < width; k++ {
244	octet = parser.raw_buffer[parser.raw_buffer_pos+k]
245
246	// Check if the octet is valid.
247	if (octet & 0xC0) != 0x80 {
248	return yaml_parser_set_reader_error(parser,
249	"invalid trailing UTF-8 octet",
250	parser.offset+k, int(octet))
251	}
252
253	// Decode the octet.
254	value = (value << 6) + rune(octet&0x3F)
255	}
256
257	// Check the length of the sequence against the value.
258	switch {
259	case width == 1:
260	case width == 2 && value >= 0x80:
261	case width == 3 && value >= 0x800:
262	case width == 4 && value >= 0x10000:
263	default:
264	return yaml_parser_set_reader_error(parser,
265	"invalid length of a UTF-8 sequence",
266	parser.offset, -1)
267	}
268
269	// Check the range of the value.
270	if value >= 0xD800 && value <= 0xDFFF \|\| value > 0x10FFFF {
271	return yaml_parser_set_reader_error(parser,
272	"invalid Unicode character",
273	parser.offset, int(value))
274	}
275
276	case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
277	var low, high int
278	if parser.encoding == yaml_UTF16LE_ENCODING {
279	low, high = 0, 1
280	} else {
281	low, high = 1, 0
282	}
283
284	// The UTF-16 encoding is not as simple as one might
285	// naively think. Check RFC 2781
286	// (http://www.ietf.org/rfc/rfc2781.txt).
287	//
288	// Normally, two subsequent bytes describe a Unicode
289	// character. However a special technique (called a
290	// surrogate pair) is used for specifying character
291	// values larger than 0xFFFF.
292	//
293	// A surrogate pair consists of two pseudo-characters:
294	// high surrogate area (0xD800-0xDBFF)
295	// low surrogate area (0xDC00-0xDFFF)
296	//
297	// The following formulas are used for decoding
298	// and encoding characters using surrogate pairs:
299	//
300	// U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
301	// U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
302	// W1 = 110110yyyyyyyyyy
303	// W2 = 110111xxxxxxxxxx
304	//
305	// where U is the character value, W1 is the high surrogate
306	// area, W2 is the low surrogate area.
307
308	// Check for incomplete UTF-16 character.
309	if raw_unread < 2 {
310	if parser.eof {
311	return yaml_parser_set_reader_error(parser,
312	"incomplete UTF-16 character",
313	parser.offset, -1)
314	}
315	break inner
316	}
317
318	// Get the character.
319	value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
320	(rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)
321
322	// Check for unexpected low surrogate area.
323	if value&0xFC00 == 0xDC00 {
324	return yaml_parser_set_reader_error(parser,
325	"unexpected low surrogate area",
326	parser.offset, int(value))
327	}
328
329	// Check for a high surrogate area.
330	if value&0xFC00 == 0xD800 {
331	width = 4
332
333	// Check for incomplete surrogate pair.
334	if raw_unread < 4 {
335	if parser.eof {
336	return yaml_parser_set_reader_error(parser,
337	"incomplete UTF-16 surrogate pair",
338	parser.offset, -1)
339	}
340	break inner
341	}
342
343	// Get the next character.
344	value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
345	(rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)
346
347	// Check for a low surrogate area.
348	if value2&0xFC00 != 0xDC00 {
349	return yaml_parser_set_reader_error(parser,
350	"expected low surrogate area",
351	parser.offset+2, int(value2))
352	}
353
354	// Generate the value of the surrogate pair.
355	value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
356	} else {
357	width = 2
358	}
359
360	default:
361	panic("impossible")
362	}
363
364	// Check if the character is in the allowed range:
365	// #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
366	// \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
367	// \| [#x10000-#x10FFFF] (32 bit)
368	switch {
369	case value == 0x09:
370	case value == 0x0A:
371	case value == 0x0D:
372	case value >= 0x20 && value <= 0x7E:
373	case value == 0x85:
374	case value >= 0xA0 && value <= 0xD7FF:
375	case value >= 0xE000 && value <= 0xFFFD:
376	case value >= 0x10000 && value <= 0x10FFFF:
377	default:
378	return yaml_parser_set_reader_error(parser,
379	"control characters are not allowed",
380	parser.offset, int(value))
381	}
382
383	// Move the raw pointers.
384	parser.raw_buffer_pos += width
385	parser.offset += width
386
387	// Finally put the character into the buffer.
388	if value <= 0x7F {
389	// 0000 0000-0000 007F . 0xxxxxxx
390	parser.buffer[buffer_len+0] = byte(value)
391	buffer_len += 1
392	} else if value <= 0x7FF {
393	// 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
394	parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
395	parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
396	buffer_len += 2
397	} else if value <= 0xFFFF {
398	// 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
399	parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
400	parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
401	parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
402	buffer_len += 3
403	} else {
404	// 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
405	parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
406	parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
407	parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
408	parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
409	buffer_len += 4
410	}
411
412	parser.unread++
413	}
414
415	// On EOF, put NUL into the buffer and return.
416	if parser.eof {
417	parser.buffer[buffer_len] = 0
418	buffer_len++
419	parser.unread++
420	break
421	}
422	}
423	// [Go] Read the documentation of this function above. To return true,
424	// we need to have the given length in the buffer. Not doing that means
425	// every single check that calls this function to make sure the buffer
426	// has a given length is Go) panicking; or C) accessing invalid memory.
427	// This happens here due to the EOF above breaking early.
428	for buffer_len < length {
429	parser.buffer[buffer_len] = 0
430	buffer_len++
431	}
432	parser.buffer = parser.buffer[:buffer_len]
433	return true
434	}