1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
|
func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
t := &transferReader{RequestMethod: "GET"}
// Unify input
isResponse := false
switch rr := msg.(type) { // 消息为响应时的赋值
case *Response:
t.Header = rr.Header
t.StatusCode = rr.StatusCode
t.ProtoMajor = rr.ProtoMajor
t.ProtoMinor = rr.ProtoMinor // 响应中不需要Connection首部字段,下面函数最后一个参数设置为true,删除该首部字段
t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true)
isResponse = true
if rr.Request != nil {
t.RequestMethod = rr.Request.Method
} // 消息为请求时的赋值
case *Request:
t.Header = rr.Header
t.RequestMethod = rr.Method
t.ProtoMajor = rr.ProtoMajor
t.ProtoMinor = rr.ProtoMinor
// Transfer semantics for Requests are exactly like those for
// Responses with status code 200, responding to a GET method
t.StatusCode = 200
t.Close = rr.Close
default:
panic("unexpected type")
}
// Default to HTTP/1.1
if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
t.ProtoMajor, t.ProtoMinor = 1, 1
}
// 处理"Transfer-Encoding"首部
err = t.fixTransferEncoding()
if err != nil {
return err
}
// 处理"Content-Length"首部,注意此处返回的是真实的消息载体长度
realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
if err != nil {
return err
} // 如果该消息为响应且对应的请求方法为HEAD,如果响应首部包含Content-Length字段,则将此作为响应的ContentLength的值,表示server // 可以接收到的数据的最大长度,由于该响应没有有效载体,此时不能使用fixLength返回的真实长度0
if isResponse && t.RequestMethod == "HEAD" {
if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
return err
} else {
t.ContentLength = n
}
} else {
t.ContentLength = realLength
}
// 处理Trailer首部字段,主要进行有消息校验
t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
if err != nil {
return err
}
// If there is no Content-Length or chunked Transfer-Encoding on a *Response
// and the status is not 1xx, 204 or 304, then the body is unbounded.
// See RFC 7230, section 3.3. // 含body但不是chunked且不包含length字段的响应称为unbounded(无法衡量长度的消息)消息,根据RFC 7230会被关闭
switch msg.(type) {
case *Response:
if realLength == -1 &&
!chunked(t.TransferEncoding) &&
bodyAllowedForStatus(t.StatusCode) {
// Unbounded body.
t.Close = true
}
}
// Prepare body reader. ContentLength < 0 means chunked encoding
// or close connection when finished, since multipart is not supported yet // 给t.Body赋值
switch { // chunked 场景处理
case chunked(t.TransferEncoding): // 如果请求为HEAD或响应状态码为1xx, 204 or 304,则消息不包含有效载体
if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) {
t.Body = NoBody
} else { // 下面会创建chunkedReader
t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close}
}
case realLength == 0:
t.Body = NoBody // 非chunked且包含有效载体(对应Content-Length),创建limitReader
case realLength > 0:
t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close}
default:
// realLength < 0, i.e. "Content-Length" not mentioned in header // 此处对于消息有效载体unbounded场景,断开底层连接
if t.Close {
// Close semantics (i.e. HTTP/1.0)
t.Body = &body{src: r, closing: t.Close}
} else {
// Persistent connection (i.e. HTTP/1.1) 好像走不到该分支。。。
t.Body = NoBody
}
}
// 为请求/响应结构体赋值并通过指针返回
switch rr := msg.(type) {
case *Request:
rr.Body = t.Body
rr.ContentLength = t.ContentLength
rr.TransferEncoding = t.TransferEncoding
rr.Close = t.Close
rr.Trailer = t.Trailer
case *Response:
rr.Body = t.Body
rr.ContentLength = t.ContentLength
rr.TransferEncoding = t.TransferEncoding
rr.Close = t.Close
rr.Trailer = t.Trailer
}
return nil
}
// 1.13.3版本的本函数描述有误,下面代码来自最新master分支func (t *transferReader) fixTransferEncoding() error { // 本函数主要处理"Transfer-Encoding"首部,如果不存在,则直接退出
raw, present := t.Header["Transfer-Encoding"]
if !present {
return nil
}
delete(t.Header, "Transfer-Encoding")
// Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. // HTTP/1.0不处理此首部
if !t.protoAtLeast(1, 1) {
return nil
}
// "Transfer-Encoding"首部字段使用逗号分割
encodings := strings.Split(raw[0], ",")
te := make([]string, 0, len(encodings))
// When adding new encodings, please maintain the invariant:
// if chunked encoding is present, it must always
// come last and it must be applied only once.
// See RFC 7230 Section 3.3.1 Transfer-Encoding. // 循环处理各个传输编码,目前仅实现了"chunked"
for i, encoding := range encodings {
encoding = strings.ToLower(strings.TrimSpace(encoding))
if encoding == "identity" {
// "identity" should not be mixed with other transfer-encodings/compressions
// because it means "no compression, no transformation".
if len(encodings) != 1 {
return &badStringError{`"identity" when present must be the only transfer encoding`, strings.Join(encodings, ",")}
}
// "identity" is not recorded.
break
}
switch {
case encoding == "chunked":
// "chunked" MUST ALWAYS be the last
// encoding as per the loop invariant.
// That is:
// Invalid: [chunked, gzip]
// Valid: [gzip, chunked]
if i+1 != len(encodings) {
return &badStringError{"chunked must be applied only once, as the last encoding", strings.Join(encodings, ",")}
}
// Supported otherwise.
case isGzipTransferEncoding(encoding):
// Supported
default:
return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", encoding)}
}
te = te[0 : len(te)+1]
te[len(te)-1] = encoding
}
if len(te) > 0 {
// RFC 7230 3.3.2 says "A sender MUST NOT send a
// Content-Length header field in any message that
// contains a Transfer-Encoding header field."
//
// but also:
// "If a message is received with both a
// Transfer-Encoding and a Content-Length header
// field, the Transfer-Encoding overrides the
// Content-Length. Such a message might indicate an
// attempt to perform request smuggling (Section 9.5)
// or response splitting (Section 9.4) and ought to be
// handled as an error. A sender MUST remove the
// received Content-Length field prior to forwarding
// such a message downstream."
//
// Reportedly, these appear in the wild. // "Transfer-Encoding"就是为了解决"Content-Length"不存在才出现了,因此当存在"Transfer-Encoding"时无需处理"Content-Length", // 此处删除"Content-Length"首部,不在fixLength函数中处理
delete(t.Header, "Content-Length")
t.TransferEncoding = te
return nil
}
return nil
}
// 本函数处理Content-Length首部,并返回真实的消息载体长度func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
isRequest := !isResponse
contentLens := header["Content-Length"]
// Hardening against HTTP request smuggling
if len(contentLens) > 1 {
// Per RFC 7230 Section 3.3.2, prevent multiple
// Content-Length headers if they differ in value.
// If there are dups of the value, remove the dups.
// See Issue 16490. // 下面按照RFC 7230的建议进行处理,如果一个Content-Length包含多个不同的value,则认为该消息无效
first := strings.TrimSpace(contentLens[0])
for _, ct := range contentLens[1:] {
if first != strings.TrimSpace(ct) {
return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens)
}
}
// 如果一个Content-Length包含多个相同的value,则仅保留一个
header.Del("Content-Length")
header.Add("Content-Length", first)
contentLens = header["Content-Length"]
}
// 处理HEAD请求
if noResponseBodyExpected(requestMethod) {
// For HTTP requests, as part of hardening against request
// smuggling (RFC 7230), don't allow a Content-Length header for
// methods which don't permit bodies. As an exception, allow
// exactly one Content-Length header if its value is "0". // 当HEAD请求中的Content-Length为0时允许存在该字段
if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") {
return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens)
}
return 0, nil
} // 处理状态码为1xx的响应,不包含消息体
if status/100 == 1 {
return 0, nil
} // 处理状态码为204和304的响应,不包含消息体
switch status {
case 204, 304:
return 0, nil
}
// 包含Transfer-Encoding时无法衡量数据长度,以Transfer-Encoding为准,设置返回长度为-1,直接返回
if chunked(te) {
return -1, nil
}
var cl string // 获取Content-Length字段值
if len(contentLens) == 1 {
cl = strings.TrimSpace(contentLens[0])
} // 对Content-Length字段的值进行有效性验证,如果有效则返回该值的整型,无效返回错误
if cl != "" {
n, err := parseContentLength(cl)
if err != nil {
return -1, err
}
return n, nil
} // 数值为空,删除该首部字段
header.Del("Content-Length")
// 请求中没有Content-Length且没有Transfer-Encoding字段的请求被认为没有有效载体
if isRequest {
// RFC 7230 neither explicitly permits nor forbids an
// entity-body on a GET request so we permit one if
// declared, but we default to 0 here (not -1 below)
// if there's no mention of a body.
// Likewise, all other request methods are assumed to have
// no body if neither Transfer-Encoding chunked nor a
// Content-Length are set.
return 0, nil
}
// Body-EOF logic based on other methods (like closing, or chunked coding) // 消息为响应,该场景后续会在readTransfer被close处理
return -1, nil
}
func (cr *connReader) startBackgroundRead() {
cr.lock()
defer cr.unlock() // 表示该连接正在被读取
if cr.inRead {
panic("invalid concurrent Body.Read call")
} // 表示该连接上是否还有数据
if cr.hasByte {
return
}
cr.inRead = true // 设置底层连接deadline为1<<64 -1
cr.conn.rwc.SetReadDeadline(time.Time{}) // 在新的goroutine中等待数据
go cr.backgroundRead()
}
func (cr *connReader) backgroundRead() { // 阻塞等待读取一个字节的数
n, err := cr.conn.rwc.Read(cr.byteBuf[:])
cr.lock() // 如果存在数据则设置cr.hasByte为true,byteBuf容量为1
if n == 1 {
cr.hasByte = true
// We were past the end of the previous request's body already
// (since we wouldn't be in a background read otherwise), so
// this is a pipelined HTTP request. Prior to Go 1.11 we used to
// send on the CloseNotify channel and cancel the context here,
// but the behavior was documented as only "may", and we only
// did that because that's how CloseNotify accidentally behaved
// in very early Go releases prior to context support. Once we
// added context support, people used a Handler's
// Request.Context() and passed it along. Having that context
// cancel on pipelined HTTP requests caused problems.
// Fortunately, almost nothing uses HTTP/1.x pipelining.
// Unfortunately, apt-get does, or sometimes does.
// New Go 1.11 behavior: don't fire CloseNotify or cancel
// contexts on pipelined requests. Shouldn't affect people, but
// fixes cases like Issue 23921. This does mean that a client
// closing their TCP connection after sending a pipelined
// request won't cancel the context, but we'll catch that on any
// write failure (in checkConnErrorWriter.Write).
// If the server never writes, yes, there are still contrived
// server & client behaviors where this fails to ever cancel the
// context, but that's kinda why HTTP/1.x pipelining died
// anyway.
}
if ne, ok := err.(net.Error); ok && cr.aborted && ne.Timeout() {
// Ignore this error. It's the expected error from
// another goroutine calling abortPendingRead.
} else if err != nil {
cr.handleReadError(err)
}
cr.aborted = false
cr.inRead = false
cr.unlock() // 当有数据时,通知cr.cond.Wait解锁
cr.cond.Broadcast()
}
|