From c6e4852be3d3cce02f3b4878575a494f034a8abf Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 17 Dec 2015 16:29:15 +0800 Subject: [PATCH 1/7] implements the writev. --- src/net/fd_unix.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++ src/net/net.go | 11 +++++++++ 2 files changed, 70 insertions(+) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 6463b0df435376..394543b9d74230 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "syscall" "time" + "unsafe" ) // Network file descriptor. @@ -333,6 +334,64 @@ func (fd *netFD) Write(p []byte) (nn int, err error) { return nn, err } +func (fd *netFD) Writev(p [][]byte) (nn int, err error) { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if err := fd.pd.PrepareWrite(); err != nil { + return 0, err + } + // convert to iovec for writev. + iovecs := make([]syscall.Iovec, len(p)) + var total int + for i, iovec := range p { + iovecs[i] = syscall.Iovec{&iovec[0], uint64(len(iovec))} + total += len(iovec) + } + for { + // send from last work point. + var index int + var left int = nn + for i, iov := range p { + if left = left - len(iov); left < 0 { + index = i + break + } + } + iovec := p[index] + iovec = iovec[len(iovec) + left:] + iovecs[index] = syscall.Iovec{&iovec[0], uint64(len(iovec))} + // to ptr and len. + ptr := uintptr(unsafe.Pointer(&iovecs[index])) + nbPtr := uintptr(len(iovecs) - index) + var n int + r0, _, e0 := syscall.Syscall(syscall.SYS_WRITEV, uintptr(fd.sysfd), ptr, nbPtr) + if n = int(r0); n > 0 { + nn += n + } + if nn == total { + break + } + if err = syscall.Errno(e0); err == syscall.EAGAIN { + if err = fd.pd.WaitWrite(); err == nil { + continue + } + } + if err != nil { + break + } + if n == 0 { + err = io.ErrUnexpectedEOF + break + } + } + if _, ok := err.(syscall.Errno); ok { + err = os.NewSyscallError("write", err) + } + return nn, err +} + func (fd *netFD) writeTo(p []byte, sa syscall.Sockaddr) (n int, err error) { if err := fd.writeLock(); err != nil { return 0, err diff --git a/src/net/net.go b/src/net/net.go index 6e84c3a100e5af..5e68d1a712880f 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -188,6 +188,17 @@ func (c *conn) Write(b []byte) (int, error) { return n, err } +func (c *conn) Writev(b [][]byte) (int, error) { + if !c.ok() { + return 0,syscall.EINVAL + } + n,err := c.fd.Writev(b) + if err != nil { + err = &OpError{Op: "writev", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n,err +} + // Close closes the connection. func (c *conn) Close() error { if !c.ok() { From 9c0717cf93766d84e8fe83b2b59a4fd64297ffd5 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 17 Dec 2015 17:32:41 +0800 Subject: [PATCH 2/7] change the api to pass the API Check. --- api/next.txt | 1 + src/net/fd_unix.go | 5 ++++- src/net/net.go | 2 +- src/net/tcpsock_posix.go | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api/next.txt b/api/next.txt index e69de29bb2d1d6..bc38ac21e8c8ad 100644 --- a/api/next.txt +++ b/api/next.txt @@ -0,0 +1 @@ +pkg net, method (*TCPConn) Writev([][]uint8) (int, error) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 394543b9d74230..945b81f95e7aa5 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -367,13 +367,16 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { nbPtr := uintptr(len(iovecs) - index) var n int r0, _, e0 := syscall.Syscall(syscall.SYS_WRITEV, uintptr(fd.sysfd), ptr, nbPtr) + if e0 != 0 { + err = syscall.Errno(e0) + } if n = int(r0); n > 0 { nn += n } if nn == total { break } - if err = syscall.Errno(e0); err == syscall.EAGAIN { + if err == syscall.EAGAIN { if err = fd.pd.WaitWrite(); err == nil { continue } diff --git a/src/net/net.go b/src/net/net.go index 5e68d1a712880f..6162e851db0f87 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -188,7 +188,7 @@ func (c *conn) Write(b []byte) (int, error) { return n, err } -func (c *conn) Writev(b [][]byte) (int, error) { +func (c *conn) writev(b [][]byte) (int, error) { if !c.ok() { return 0,syscall.EINVAL } diff --git a/src/net/tcpsock_posix.go b/src/net/tcpsock_posix.go index 7e49b769e1c98c..4ee2d0cfec4ee9 100644 --- a/src/net/tcpsock_posix.go +++ b/src/net/tcpsock_posix.go @@ -52,6 +52,10 @@ func newTCPConn(fd *netFD) *TCPConn { return c } +func (c *TCPConn) Writev(b [][]byte) (n int, err error) { + return c.conn.writev(b) +} + // ReadFrom implements the io.ReaderFrom ReadFrom method. func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { if n, err, handled := sendFile(c.fd, r); handled { From 34eb9de7d563332d526b48457e3908b203116711 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 17 Dec 2015 17:37:21 +0800 Subject: [PATCH 3/7] change the api from next to 1.5 --- api/go1.5.txt | 1 + api/next.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/api/go1.5.txt b/api/go1.5.txt index d9cf7970755a40..c96db7eb9b4efb 100644 --- a/api/go1.5.txt +++ b/api/go1.5.txt @@ -871,6 +871,7 @@ pkg net/mail, type AddressParser struct, WordDecoder *mime.WordDecoder pkg net/smtp, method (*Client) TLSConnectionState() (tls.ConnectionState, bool) pkg net/url, method (*URL) EscapedPath() string pkg net/url, type URL struct, RawPath string +pkg net, method (*TCPConn) Writev([][]uint8) (int, error) pkg os, func LookupEnv(string) (string, bool) pkg os/signal, func Ignore(...os.Signal) pkg os/signal, func Reset(...os.Signal) diff --git a/api/next.txt b/api/next.txt index bc38ac21e8c8ad..e69de29bb2d1d6 100644 --- a/api/next.txt +++ b/api/next.txt @@ -1 +0,0 @@ -pkg net, method (*TCPConn) Writev([][]uint8) (int, error) From 568e04a524282e5253a110e5eccd716dd3a16655 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 18 Dec 2015 10:40:28 +0800 Subject: [PATCH 4/7] add print log --- src/net/fd_unix.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 945b81f95e7aa5..11aed69575d407 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -14,6 +14,7 @@ import ( "syscall" "time" "unsafe" + //"fmt" ) // Network file descriptor. @@ -349,6 +350,7 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { iovecs[i] = syscall.Iovec{&iovec[0], uint64(len(iovec))} total += len(iovec) } + //fmt.Println(fmt.Sprintf("writev %v iovecs, total %v bytes", len(p), total)) for { // send from last work point. var index int @@ -365,6 +367,11 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { // to ptr and len. ptr := uintptr(unsafe.Pointer(&iovecs[index])) nbPtr := uintptr(len(iovecs) - index) + // max iovec to send is 1024 + // TODO: FIXME: read from sysconf. + if int(nbPtr) > 1024 { + nbPtr = uintptr(1024) + } var n int r0, _, e0 := syscall.Syscall(syscall.SYS_WRITEV, uintptr(fd.sysfd), ptr, nbPtr) if e0 != 0 { @@ -373,6 +380,7 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { if n = int(r0); n > 0 { nn += n } + //fmt.Println(fmt.Sprintf(" %v:%v %v/%v %v", index, left, nn, total, err)) if nn == total { break } From 0c0423ba4912d0787bf6de8b676e84b079e547ef Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 18 Dec 2015 13:12:21 +0800 Subject: [PATCH 5/7] cache the iovecs for writev. --- src/net/fd_unix.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 11aed69575d407..b0c75f31e4dd6c 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -31,6 +31,9 @@ type netFD struct { laddr Addr raddr Addr + // writev cache. + iovecs []syscall.Iovec + // wait server pd pollDesc } @@ -344,10 +347,14 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { return 0, err } // convert to iovec for writev. - iovecs := make([]syscall.Iovec, len(p)) + if fd.iovecs == nil { + fd.iovecs = make([]syscall.Iovec, 0) + } + iovecs := fd.iovecs[0:0] + // total bytes to sent. var total int - for i, iovec := range p { - iovecs[i] = syscall.Iovec{&iovec[0], uint64(len(iovec))} + for _, iovec := range p { + iovecs = append(iovecs, syscall.Iovec{&iovec[0], uint64(len(iovec))}) total += len(iovec) } //fmt.Println(fmt.Sprintf("writev %v iovecs, total %v bytes", len(p), total)) From 5eb6b41d1e44c199ffd18e972ba93dde8cd5bfe6 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 18 Dec 2015 15:40:54 +0800 Subject: [PATCH 6/7] refine comments. --- src/net/fd_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index b0c75f31e4dd6c..071c5f645b9309 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -387,7 +387,7 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { if n = int(r0); n > 0 { nn += n } - //fmt.Println(fmt.Sprintf(" %v:%v %v/%v %v", index, left, nn, total, err)) + //fmt.Println(fmt.Sprintf(" %v:%v:%v %v/%v %v", index, left, len(p[index]) - len(iovec), nn, total, err)) if nn == total { break } From ea101b395b0c005c03028eb60c57762ed390e6e6 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 18 Dec 2015 16:17:38 +0800 Subject: [PATCH 7/7] avoid alloc Iovec. --- src/net/fd_unix.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 071c5f645b9309..d90ca98f40cc5a 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -32,7 +32,7 @@ type netFD struct { raddr Addr // writev cache. - iovecs []syscall.Iovec + iovecs []syscall.Iovec // wait server pd pollDesc @@ -347,14 +347,15 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { return 0, err } // convert to iovec for writev. - if fd.iovecs == nil { - fd.iovecs = make([]syscall.Iovec, 0) + if cap(fd.iovecs) < len(p) { + fd.iovecs = make([]syscall.Iovec, 0, 2*len(p)) } - iovecs := fd.iovecs[0:0] + iovecs := fd.iovecs[0:len(p)] // total bytes to sent. var total int - for _, iovec := range p { - iovecs = append(iovecs, syscall.Iovec{&iovec[0], uint64(len(iovec))}) + for i, iovec := range p { + iovecs[i].Base = &iovec[0] + iovecs[i].Len = uint64(len(iovec)) total += len(iovec) } //fmt.Println(fmt.Sprintf("writev %v iovecs, total %v bytes", len(p), total)) @@ -368,9 +369,9 @@ func (fd *netFD) Writev(p [][]byte) (nn int, err error) { break } } - iovec := p[index] - iovec = iovec[len(iovec) + left:] - iovecs[index] = syscall.Iovec{&iovec[0], uint64(len(iovec))} + iovec := p[index][len(p[index])+left:] + iovecs[index].Base = &iovec[0] + iovecs[index].Len = uint64(len(iovec)) // to ptr and len. ptr := uintptr(unsafe.Pointer(&iovecs[index])) nbPtr := uintptr(len(iovecs) - index)