1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
use std::io::{self, BufRead};
use std::error::Error;
use std::fmt;
use std::str;
use super::*;
/// Wraps a `std::io::BufRead` buffered byte stream and decode it as UTF-8.
pub struct BufReadDecoder<B: BufRead> {
buf_read: B,
bytes_consumed: usize,
incomplete: Incomplete,
}
#[derive(Debug)]
pub enum BufReadDecoderError<'a> {
/// Represents one UTF-8 error in the byte stream.
///
/// In lossy decoding, each such error should be replaced with U+FFFD.
/// (See `BufReadDecoder::next_lossy` and `BufReadDecoderError::lossy`.)
InvalidByteSequence(&'a [u8]),
/// An I/O error from the underlying byte stream
Io(io::Error),
}
impl<'a> BufReadDecoderError<'a> {
/// Replace UTF-8 errors with U+FFFD
pub fn lossy(self) -> Result<&'static str, io::Error> {
match self {
BufReadDecoderError::Io(error) => Err(error),
BufReadDecoderError::InvalidByteSequence(_) => Ok(REPLACEMENT_CHARACTER),
}
}
}
impl<'a> fmt::Display for BufReadDecoderError<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
BufReadDecoderError::InvalidByteSequence(bytes) => {
write!(f, "invalid byte sequence: {:02x?}", bytes)
}
BufReadDecoderError::Io(ref err) => write!(f, "underlying bytestream error: {}", err),
}
}
}
impl<'a> Error for BufReadDecoderError<'a> {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match *self {
BufReadDecoderError::InvalidByteSequence(_) => None,
BufReadDecoderError::Io(ref err) => Some(err),
}
}
}
impl<B: BufRead> BufReadDecoder<B> {
/// This is to `Read::read_to_string` what `String::from_utf8_lossy` is to `String::from_utf8`.
pub fn read_to_string_lossy(buf_read: B) -> io::Result<String> {
let mut decoder = Self::new(buf_read);
let mut string = String::new();
while let Some(result) = decoder.next_lossy() {
string.push_str(result?)
}
Ok(string)
}
pub fn new(buf_read: B) -> Self {
Self {
buf_read,
bytes_consumed: 0,
incomplete: Incomplete::empty(),
}
}
/// Same as `BufReadDecoder::next_strict`, but replace UTF-8 errors with U+FFFD.
pub fn next_lossy(&mut self) -> Option<io::Result<&str>> {
self.next_strict().map(|result| result.or_else(|e| e.lossy()))
}
/// Decode and consume the next chunk of UTF-8 input.
///
/// This method is intended to be called repeatedly until it returns `None`,
/// which represents EOF from the underlying byte stream.
/// This is similar to `Iterator::next`,
/// except that decoded chunks borrow the decoder (~iterator)
/// so they need to be handled or copied before the next chunk can start decoding.
pub fn next_strict(&mut self) -> Option<Result<&str, BufReadDecoderError>> {
enum BytesSource {
BufRead(usize),
Incomplete,
}
macro_rules! try_io {
($io_result: expr) => {
match $io_result {
Ok(value) => value,
Err(error) => return Some(Err(BufReadDecoderError::Io(error)))
}
}
}
let (source, result) = loop {
if self.bytes_consumed > 0 {
self.buf_read.consume(self.bytes_consumed);
self.bytes_consumed = 0;
}
let buf = try_io!(self.buf_read.fill_buf());
// Force loop iteration to go through an explicit `continue`
enum Unreachable {}
let _: Unreachable = if self.incomplete.is_empty() {
if buf.is_empty() {
return None // EOF
}
match str::from_utf8(buf) {
Ok(_) => {
break (BytesSource::BufRead(buf.len()), Ok(()))
}
Err(error) => {
let valid_up_to = error.valid_up_to();
if valid_up_to > 0 {
break (BytesSource::BufRead(valid_up_to), Ok(()))
}
match error.error_len() {
Some(invalid_sequence_length) => {
break (BytesSource::BufRead(invalid_sequence_length), Err(()))
}
None => {
self.bytes_consumed = buf.len();
self.incomplete = Incomplete::new(buf);
// need more input bytes
continue
}
}
}
}
} else {
if buf.is_empty() {
break (BytesSource::Incomplete, Err(())) // EOF with incomplete code point
}
let (consumed, opt_result) = self.incomplete.try_complete_offsets(buf);
self.bytes_consumed = consumed;
match opt_result {
None => {
// need more input bytes
continue
}
Some(result) => {
break (BytesSource::Incomplete, result)
}
}
};
};
let bytes = match source {
BytesSource::BufRead(byte_count) => {
self.bytes_consumed = byte_count;
let buf = try_io!(self.buf_read.fill_buf());
&buf[..byte_count]
}
BytesSource::Incomplete => {
self.incomplete.take_buffer()
}
};
match result {
Ok(()) => Some(Ok(unsafe { str::from_utf8_unchecked(bytes) })),
Err(()) => Some(Err(BufReadDecoderError::InvalidByteSequence(bytes))),
}
}
}