gopkg.in/yaml.v3/readerc.go

// 
// Copyright (c) 2011-2019 Canonical Ltd
// Copyright (c) 2006-2010 Kirill Simonov
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package yaml

import (
        "io"
)

// Set the reader error and return 0.
func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
        parser.error = yaml_READER_ERROR
        parser.problem = problem
        parser.problem_offset = offset
        parser.problem_value = value
        return false
}

// Byte order marks.
const (
        bom_UTF8    = "\xef\xbb\xbf"
        bom_UTF16LE = "\xff\xfe"
        bom_UTF16BE = "\xfe\xff"
)

// Determine the input stream encoding by checking the BOM symbol. If no BOM is
// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
        // Ensure that we had enough bytes in the raw buffer.
        for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
                if !yaml_parser_update_raw_buffer(parser) {
                        return false
                }
        }

        // Determine the encoding.
        buf := parser.raw_buffer
        pos := parser.raw_buffer_pos
        avail := len(buf) - pos
        if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
                parser.encoding = yaml_UTF16LE_ENCODING
                parser.raw_buffer_pos += 2
                parser.offset += 2
        } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
                parser.encoding = yaml_UTF16BE_ENCODING
                parser.raw_buffer_pos += 2
                parser.offset += 2
        } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
                parser.encoding = yaml_UTF8_ENCODING
                parser.raw_buffer_pos += 3
                parser.offset += 3
        } else {
                parser.encoding = yaml_UTF8_ENCODING
        }
        return true
}

// Update the raw buffer.
func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
        size_read := 0

        // Return if the raw buffer is full.
        if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
                return true
        }

        // Return on EOF.
        if parser.eof {
                return true
        }

        // Move the remaining bytes in the raw buffer to the beginning.
        if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
                copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
        }
        parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
        parser.raw_buffer_pos = 0

        // Call the read handler to fill the buffer.
        size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
        parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
        if err == io.EOF {
                parser.eof = true
        } else if err != nil {
                return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
        }
        return true
}

// Ensure that the buffer contains at least `length` characters.
// Return true on success, false on failure.
//
// The length is supposed to be significantly less that the buffer size.
func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
        if parser.read_handler == nil {
                panic("read handler must be set")
        }

        // [Go] This function was changed to guarantee the requested length size at EOF.
        // The fact we need to do this is pretty awful, but the description above implies
        // for that to be the case, and there are tests

        // If the EOF flag is set and the raw buffer is empty, do nothing.
        if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
                // [Go] ACTUALLY! Read the documentation of this function above.
                // This is just broken. To return true, we need to have the
                // given length in the buffer. Not doing that means every single
                // check that calls this function to make sure the buffer has a
                // given length is Go) panicking; or C) accessing invalid memory.
                //return true
        }

        // Return if the buffer contains enough characters.
        if parser.unread >= length {
                return true
        }

        // Determine the input encoding if it is not known yet.
        if parser.encoding == yaml_ANY_ENCODING {
                if !yaml_parser_determine_encoding(parser) {
                        return false
                }
        }

        // Move the unread characters to the beginning of the buffer.
        buffer_len := len(parser.buffer)
        if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
                copy(parser.buffer, parser.buffer[parser.buffer_pos:])
                buffer_len -= parser.buffer_pos
                parser.buffer_pos = 0
        } else if parser.buffer_pos == buffer_len {
                buffer_len = 0
                parser.buffer_pos = 0
        }

        // Open the whole buffer for writing, and cut it before returning.
        parser.buffer = parser.buffer[:cap(parser.buffer)]

        // Fill the buffer until it has enough characters.
        first := true
        for parser.unread < length {

                // Fill the raw buffer if necessary.
                if !first || parser.raw_buffer_pos == len(parser.raw_buffer) {
                        if !yaml_parser_update_raw_buffer(parser) {
                                parser.buffer = parser.buffer[:buffer_len]
                                return false
                        }
                }
                first = false

                // Decode the raw buffer.
        inner:
                for parser.raw_buffer_pos != len(parser.raw_buffer) {
                        var value rune
                        var width int

                        raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos

                        // Decode the next character.
                        switch parser.encoding {
                        case yaml_UTF8_ENCODING:
                                // Decode a UTF-8 character.  Check RFC 3629
                                // (http://www.ietf.org/rfc/rfc3629.txt) for more details.
                                //
                                // The following table (taken from the RFC) is used for
                                // decoding.
                                //
                                //    Char. number range |        UTF-8 octet sequence
                                //      (hexadecimal)    |              (binary)
                                //   --------------------+------------------------------------
                                //   0000 0000-0000 007F | 0xxxxxxx
                                //   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
                                //   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
                                //   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                                //
                                // Additionally, the characters in the range 0xD800-0xDFFF
                                // are prohibited as they are reserved for use with UTF-16
                                // surrogate pairs.

                                // Determine the length of the UTF-8 sequence.
                                octet := parser.raw_buffer[parser.raw_buffer_pos]
                                switch {
                                case octet&0x80 == 0x00:
                                        width = 1
                                case octet&0xE0 == 0xC0:
                                        width = 2
                                case octet&0xF0 == 0xE0:
                                        width = 3
                                case octet&0xF8 == 0xF0:
                                        width = 4
                                default:
                                        // The leading octet is invalid.
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid leading UTF-8 octet",
                                                parser.offset, int(octet))
                                }

                                // Check if the raw buffer contains an incomplete character.
                                if width > raw_unread {
                                        if parser.eof {
                                                return yaml_parser_set_reader_error(parser,
                                                        "incomplete UTF-8 octet sequence",
                                                        parser.offset, -1)
                                        }
                                        break inner
                                }

                                // Decode the leading octet.
                                switch {
                                case octet&0x80 == 0x00:
                                        value = rune(octet & 0x7F)
                                case octet&0xE0 == 0xC0:
                                        value = rune(octet & 0x1F)
                                case octet&0xF0 == 0xE0:
                                        value = rune(octet & 0x0F)
                                case octet&0xF8 == 0xF0:
                                        value = rune(octet & 0x07)
                                default:
                                        value = 0
                                }

                                // Check and decode the trailing octets.
                                for k := 1; k < width; k++ {
                                        octet = parser.raw_buffer[parser.raw_buffer_pos+k]

                                        // Check if the octet is valid.
                                        if (octet & 0xC0) != 0x80 {
                                                return yaml_parser_set_reader_error(parser,
                                                        "invalid trailing UTF-8 octet",
                                                        parser.offset+k, int(octet))
                                        }

                                        // Decode the octet.
                                        value = (value << 6) + rune(octet&0x3F)
                                }

                                // Check the length of the sequence against the value.
                                switch {
                                case width == 1:
                                case width == 2 && value >= 0x80:
                                case width == 3 && value >= 0x800:
                                case width == 4 && value >= 0x10000:
                                default:
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid length of a UTF-8 sequence",
                                                parser.offset, -1)
                                }

                                // Check the range of the value.
                                if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF {
                                        return yaml_parser_set_reader_error(parser,
                                                "invalid Unicode character",
                                                parser.offset, int(value))
                                }

                        case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
                                var low, high int
                                if parser.encoding == yaml_UTF16LE_ENCODING {
                                        low, high = 0, 1
                                } else {
                                        low, high = 1, 0
                                }

                                // The UTF-16 encoding is not as simple as one might
                                // naively think.  Check RFC 2781
                                // (http://www.ietf.org/rfc/rfc2781.txt).
                                //
                                // Normally, two subsequent bytes describe a Unicode
                                // character.  However a special technique (called a
                                // surrogate pair) is used for specifying character
                                // values larger than 0xFFFF.
                                //
                                // A surrogate pair consists of two pseudo-characters:
                                //      high surrogate area (0xD800-0xDBFF)
                                //      low surrogate area (0xDC00-0xDFFF)
                                //
                                // The following formulas are used for decoding
                                // and encoding characters using surrogate pairs:
                                //
                                //  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
                                //  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
                                //  W1 = 110110yyyyyyyyyy
                                //  W2 = 110111xxxxxxxxxx
                                //
                                // where U is the character value, W1 is the high surrogate
                                // area, W2 is the low surrogate area.

                                // Check for incomplete UTF-16 character.
                                if raw_unread < 2 {
                                        if parser.eof {
                                                return yaml_parser_set_reader_error(parser,
                                                        "incomplete UTF-16 character",
                                                        parser.offset, -1)
                                        }
                                        break inner
                                }

                                // Get the character.
                                value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
                                        (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)

                                // Check for unexpected low surrogate area.
                                if value&0xFC00 == 0xDC00 {
                                        return yaml_parser_set_reader_error(parser,
                                                "unexpected low surrogate area",
                                                parser.offset, int(value))
                                }

                                // Check for a high surrogate area.
                                if value&0xFC00 == 0xD800 {
                                        width = 4

                                        // Check for incomplete surrogate pair.
                                        if raw_unread < 4 {
                                                if parser.eof {
                                                        return yaml_parser_set_reader_error(parser,
                                                                "incomplete UTF-16 surrogate pair",
                                                                parser.offset, -1)
                                                }
                                                break inner
                                        }

                                        // Get the next character.
                                        value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
                                                (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)

                                        // Check for a low surrogate area.
                                        if value2&0xFC00 != 0xDC00 {
                                                return yaml_parser_set_reader_error(parser,
                                                        "expected low surrogate area",
                                                        parser.offset+2, int(value2))
                                        }

                                        // Generate the value of the surrogate pair.
                                        value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
                                } else {
                                        width = 2
                                }

                        default:
                                panic("impossible")
                        }

                        // Check if the character is in the allowed range:
                        //      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
                        //      | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD]    (16 bit)
                        //      | [#x10000-#x10FFFF]                        (32 bit)
                        switch {
                        case value == 0x09:
                        case value == 0x0A:
                        case value == 0x0D:
                        case value >= 0x20 && value <= 0x7E:
                        case value == 0x85:
                        case value >= 0xA0 && value <= 0xD7FF:
                        case value >= 0xE000 && value <= 0xFFFD:
                        case value >= 0x10000 && value <= 0x10FFFF:
                        default:
                                return yaml_parser_set_reader_error(parser,
                                        "control characters are not allowed",
                                        parser.offset, int(value))
                        }

                        // Move the raw pointers.
                        parser.raw_buffer_pos += width
                        parser.offset += width

                        // Finally put the character into the buffer.
                        if value <= 0x7F {
                                // 0000 0000-0000 007F . 0xxxxxxx
                                parser.buffer[buffer_len+0] = byte(value)
                                buffer_len += 1
                        } else if value <= 0x7FF {
                                // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
                                parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
                                buffer_len += 2
                        } else if value <= 0xFFFF {
                                // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
                                parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
                                parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
                                buffer_len += 3
                        } else {
                                // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
                                parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
                                parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
                                parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
                                parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
                                buffer_len += 4
                        }

                        parser.unread++
                }

                // On EOF, put NUL into the buffer and return.
                if parser.eof {
                        parser.buffer[buffer_len] = 0
                        buffer_len++
                        parser.unread++
                        break
                }
        }
        // [Go] Read the documentation of this function above. To return true,
        // we need to have the given length in the buffer. Not doing that means
        // every single check that calls this function to make sure the buffer
        // has a given length is Go) panicking; or C) accessing invalid memory.
        // This happens here due to the EOF above breaking early.
        for buffer_len < length {
                parser.buffer[buffer_len] = 0
                buffer_len++
        }
        parser.buffer = parser.buffer[:buffer_len]
        return true
}
Revision:	1.1
Committed:	Mon Sep 30 00:42:06 2024 UTC (6 weeks, 4 days ago) by yakumo_izuru
Branch:	MAIN
CVS Tags:	HEAD
Log Message:	Mirrored from https://git.chaotic.ninja/git/yakumo_izuru/aya
#	User	Rev	Content
1	yakumo_izuru	1.1	//
2			// Copyright (c) 2011-2019 Canonical Ltd
3			// Copyright (c) 2006-2010 Kirill Simonov
4			//
5			// Permission is hereby granted, free of charge, to any person obtaining a copy of
6			// this software and associated documentation files (the "Software"), to deal in
7			// the Software without restriction, including without limitation the rights to
8			// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
9			// of the Software, and to permit persons to whom the Software is furnished to do
10			// so, subject to the following conditions:
11			//
12			// The above copyright notice and this permission notice shall be included in all
13			// copies or substantial portions of the Software.
14			//
15			// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16			// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17			// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18			// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19			// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20			// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21			// SOFTWARE.
22
23			package yaml
24
25			import (
26			"io"
27			)
28
29			// Set the reader error and return 0.
30			func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
31			parser.error = yaml_READER_ERROR
32			parser.problem = problem
33			parser.problem_offset = offset
34			parser.problem_value = value
35			return false
36			}
37
38			// Byte order marks.
39			const (
40			bom_UTF8 = "\xef\xbb\xbf"
41			bom_UTF16LE = "\xff\xfe"
42			bom_UTF16BE = "\xfe\xff"
43			)
44
45			// Determine the input stream encoding by checking the BOM symbol. If no BOM is
46			// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
47			func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
48			// Ensure that we had enough bytes in the raw buffer.
49			for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
50			if !yaml_parser_update_raw_buffer(parser) {
51			return false
52			}
53			}
54
55			// Determine the encoding.
56			buf := parser.raw_buffer
57			pos := parser.raw_buffer_pos
58			avail := len(buf) - pos
59			if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
60			parser.encoding = yaml_UTF16LE_ENCODING
61			parser.raw_buffer_pos += 2
62			parser.offset += 2
63			} else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
64			parser.encoding = yaml_UTF16BE_ENCODING
65			parser.raw_buffer_pos += 2
66			parser.offset += 2
67			} else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
68			parser.encoding = yaml_UTF8_ENCODING
69			parser.raw_buffer_pos += 3
70			parser.offset += 3
71			} else {
72			parser.encoding = yaml_UTF8_ENCODING
73			}
74			return true
75			}
76
77			// Update the raw buffer.
78			func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
79			size_read := 0
80
81			// Return if the raw buffer is full.
82			if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
83			return true
84			}
85
86			// Return on EOF.
87			if parser.eof {
88			return true
89			}
90
91			// Move the remaining bytes in the raw buffer to the beginning.
92			if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
93			copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
94			}
95			parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
96			parser.raw_buffer_pos = 0
97
98			// Call the read handler to fill the buffer.
99			size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
100			parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
101			if err == io.EOF {
102			parser.eof = true
103			} else if err != nil {
104			return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
105			}
106			return true
107			}
108
109			// Ensure that the buffer contains at least `length` characters.
110			// Return true on success, false on failure.
111			//
112			// The length is supposed to be significantly less that the buffer size.
113			func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
114			if parser.read_handler == nil {
115			panic("read handler must be set")
116			}
117
118			// [Go] This function was changed to guarantee the requested length size at EOF.
119			// The fact we need to do this is pretty awful, but the description above implies
120			// for that to be the case, and there are tests
121
122			// If the EOF flag is set and the raw buffer is empty, do nothing.
123			if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
124			// [Go] ACTUALLY! Read the documentation of this function above.
125			// This is just broken. To return true, we need to have the
126			// given length in the buffer. Not doing that means every single
127			// check that calls this function to make sure the buffer has a
128			// given length is Go) panicking; or C) accessing invalid memory.
129			//return true
130			}
131
132			// Return if the buffer contains enough characters.
133			if parser.unread >= length {
134			return true
135			}
136
137			// Determine the input encoding if it is not known yet.
138			if parser.encoding == yaml_ANY_ENCODING {
139			if !yaml_parser_determine_encoding(parser) {
140			return false
141			}
142			}
143
144			// Move the unread characters to the beginning of the buffer.
145			buffer_len := len(parser.buffer)
146			if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
147			copy(parser.buffer, parser.buffer[parser.buffer_pos:])
148			buffer_len -= parser.buffer_pos
149			parser.buffer_pos = 0
150			} else if parser.buffer_pos == buffer_len {
151			buffer_len = 0
152			parser.buffer_pos = 0
153			}
154
155			// Open the whole buffer for writing, and cut it before returning.
156			parser.buffer = parser.buffer[:cap(parser.buffer)]
157
158			// Fill the buffer until it has enough characters.
159			first := true
160			for parser.unread < length {
161
162			// Fill the raw buffer if necessary.
163			if !first \|\| parser.raw_buffer_pos == len(parser.raw_buffer) {
164			if !yaml_parser_update_raw_buffer(parser) {
165			parser.buffer = parser.buffer[:buffer_len]
166			return false
167			}
168			}
169			first = false
170
171			// Decode the raw buffer.
172			inner:
173			for parser.raw_buffer_pos != len(parser.raw_buffer) {
174			var value rune
175			var width int
176
177			raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos
178
179			// Decode the next character.
180			switch parser.encoding {
181			case yaml_UTF8_ENCODING:
182			// Decode a UTF-8 character. Check RFC 3629
183			// (http://www.ietf.org/rfc/rfc3629.txt) for more details.
184			//
185			// The following table (taken from the RFC) is used for
186			// decoding.
187			//
188			// Char. number range \| UTF-8 octet sequence
189			// (hexadecimal) \| (binary)
190			// --------------------+------------------------------------
191			// 0000 0000-0000 007F \| 0xxxxxxx
192			// 0000 0080-0000 07FF \| 110xxxxx 10xxxxxx
193			// 0000 0800-0000 FFFF \| 1110xxxx 10xxxxxx 10xxxxxx
194			// 0001 0000-0010 FFFF \| 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
195			//
196			// Additionally, the characters in the range 0xD800-0xDFFF
197			// are prohibited as they are reserved for use with UTF-16
198			// surrogate pairs.
199
200			// Determine the length of the UTF-8 sequence.
201			octet := parser.raw_buffer[parser.raw_buffer_pos]
202			switch {
203			case octet&0x80 == 0x00:
204			width = 1
205			case octet&0xE0 == 0xC0:
206			width = 2
207			case octet&0xF0 == 0xE0:
208			width = 3
209			case octet&0xF8 == 0xF0:
210			width = 4
211			default:
212			// The leading octet is invalid.
213			return yaml_parser_set_reader_error(parser,
214			"invalid leading UTF-8 octet",
215			parser.offset, int(octet))
216			}
217
218			// Check if the raw buffer contains an incomplete character.
219			if width > raw_unread {
220			if parser.eof {
221			return yaml_parser_set_reader_error(parser,
222			"incomplete UTF-8 octet sequence",
223			parser.offset, -1)
224			}
225			break inner
226			}
227
228			// Decode the leading octet.
229			switch {
230			case octet&0x80 == 0x00:
231			value = rune(octet & 0x7F)
232			case octet&0xE0 == 0xC0:
233			value = rune(octet & 0x1F)
234			case octet&0xF0 == 0xE0:
235			value = rune(octet & 0x0F)
236			case octet&0xF8 == 0xF0:
237			value = rune(octet & 0x07)
238			default:
239			value = 0
240			}
241
242			// Check and decode the trailing octets.
243			for k := 1; k < width; k++ {
244			octet = parser.raw_buffer[parser.raw_buffer_pos+k]
245
246			// Check if the octet is valid.
247			if (octet & 0xC0) != 0x80 {
248			return yaml_parser_set_reader_error(parser,
249			"invalid trailing UTF-8 octet",
250			parser.offset+k, int(octet))
251			}
252
253			// Decode the octet.
254			value = (value << 6) + rune(octet&0x3F)
255			}
256
257			// Check the length of the sequence against the value.
258			switch {
259			case width == 1:
260			case width == 2 && value >= 0x80:
261			case width == 3 && value >= 0x800:
262			case width == 4 && value >= 0x10000:
263			default:
264			return yaml_parser_set_reader_error(parser,
265			"invalid length of a UTF-8 sequence",
266			parser.offset, -1)
267			}
268
269			// Check the range of the value.
270			if value >= 0xD800 && value <= 0xDFFF \|\| value > 0x10FFFF {
271			return yaml_parser_set_reader_error(parser,
272			"invalid Unicode character",
273			parser.offset, int(value))
274			}
275
276			case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
277			var low, high int
278			if parser.encoding == yaml_UTF16LE_ENCODING {
279			low, high = 0, 1
280			} else {
281			low, high = 1, 0
282			}
283
284			// The UTF-16 encoding is not as simple as one might
285			// naively think. Check RFC 2781
286			// (http://www.ietf.org/rfc/rfc2781.txt).
287			//
288			// Normally, two subsequent bytes describe a Unicode
289			// character. However a special technique (called a
290			// surrogate pair) is used for specifying character
291			// values larger than 0xFFFF.
292			//
293			// A surrogate pair consists of two pseudo-characters:
294			// high surrogate area (0xD800-0xDBFF)
295			// low surrogate area (0xDC00-0xDFFF)
296			//
297			// The following formulas are used for decoding
298			// and encoding characters using surrogate pairs:
299			//
300			// U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF)
301			// U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF)
302			// W1 = 110110yyyyyyyyyy
303			// W2 = 110111xxxxxxxxxx
304			//
305			// where U is the character value, W1 is the high surrogate
306			// area, W2 is the low surrogate area.
307
308			// Check for incomplete UTF-16 character.
309			if raw_unread < 2 {
310			if parser.eof {
311			return yaml_parser_set_reader_error(parser,
312			"incomplete UTF-16 character",
313			parser.offset, -1)
314			}
315			break inner
316			}
317
318			// Get the character.
319			value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
320			(rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)
321
322			// Check for unexpected low surrogate area.
323			if value&0xFC00 == 0xDC00 {
324			return yaml_parser_set_reader_error(parser,
325			"unexpected low surrogate area",
326			parser.offset, int(value))
327			}
328
329			// Check for a high surrogate area.
330			if value&0xFC00 == 0xD800 {
331			width = 4
332
333			// Check for incomplete surrogate pair.
334			if raw_unread < 4 {
335			if parser.eof {
336			return yaml_parser_set_reader_error(parser,
337			"incomplete UTF-16 surrogate pair",
338			parser.offset, -1)
339			}
340			break inner
341			}
342
343			// Get the next character.
344			value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
345			(rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)
346
347			// Check for a low surrogate area.
348			if value2&0xFC00 != 0xDC00 {
349			return yaml_parser_set_reader_error(parser,
350			"expected low surrogate area",
351			parser.offset+2, int(value2))
352			}
353
354			// Generate the value of the surrogate pair.
355			value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
356			} else {
357			width = 2
358			}
359
360			default:
361			panic("impossible")
362			}
363
364			// Check if the character is in the allowed range:
365			// #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
366			// \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
367			// \| [#x10000-#x10FFFF] (32 bit)
368			switch {
369			case value == 0x09:
370			case value == 0x0A:
371			case value == 0x0D:
372			case value >= 0x20 && value <= 0x7E:
373			case value == 0x85:
374			case value >= 0xA0 && value <= 0xD7FF:
375			case value >= 0xE000 && value <= 0xFFFD:
376			case value >= 0x10000 && value <= 0x10FFFF:
377			default:
378			return yaml_parser_set_reader_error(parser,
379			"control characters are not allowed",
380			parser.offset, int(value))
381			}
382
383			// Move the raw pointers.
384			parser.raw_buffer_pos += width
385			parser.offset += width
386
387			// Finally put the character into the buffer.
388			if value <= 0x7F {
389			// 0000 0000-0000 007F . 0xxxxxxx
390			parser.buffer[buffer_len+0] = byte(value)
391			buffer_len += 1
392			} else if value <= 0x7FF {
393			// 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
394			parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
395			parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
396			buffer_len += 2
397			} else if value <= 0xFFFF {
398			// 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
399			parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
400			parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
401			parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
402			buffer_len += 3
403			} else {
404			// 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
405			parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
406			parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
407			parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
408			parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
409			buffer_len += 4
410			}
411
412			parser.unread++
413			}
414
415			// On EOF, put NUL into the buffer and return.
416			if parser.eof {
417			parser.buffer[buffer_len] = 0
418			buffer_len++
419			parser.unread++
420			break
421			}
422			}
423			// [Go] Read the documentation of this function above. To return true,
424			// we need to have the given length in the buffer. Not doing that means
425			// every single check that calls this function to make sure the buffer
426			// has a given length is Go) panicking; or C) accessing invalid memory.
427			// This happens here due to the EOF above breaking early.
428			for buffer_len < length {
429			parser.buffer[buffer_len] = 0
430			buffer_len++
431			}
432			parser.buffer = parser.buffer[:buffer_len]
433			return true
434			}