Develop binary based network server in Go introduction of []byte, bytes, encoding/binary, bufio etc

1. NetWork & IO
I/O and binary programming
2013/11/14
#GoCon

2. ●
●
●
●
●
●
id: Jxck
github: Jxck
twitter: jxck_
about: http://jxck.io
blog: http://d.hatena.ne.jp/jxck
Love: music
Jack

4. Go研
https://github.com/goken/goken

5. Now working on ...
● http2 server/client
○
○
○
○
github.com/Jxck/http2
github.com/Jxck/hpack
github.com/Jxck/logger
github.com/Jxck/color
● Caution
○
○
○
○
not enough error handling
not enough test
not enough api/feature
under heavy working :p

6. network server
I/O and binary programming

7. NetWork Server in Go
●
●
●
●
●
●
●
net
[]byte
bytes
encoding/binary
io
io/ioutil
bufio

8. No Error Handling in Sample Codes
func main() {
listener, err := net.Listen("tcp", "127.0.0.1:3000") // err
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept() // err
if err != nil {
log.Fatal(err)
}
defer func() {
log.Println("close connection")
conn.Close()
}()
_, err = conn.Write([]byte("hellon")) // send hello
if err != nil {
log.Fatal(err)
}
}
}
Hard To Read on Slide

9. import “net”
tcp server in go

10. TCP Server
package main
import "net"
func main() {
listener, _ := net.Listen("tcp", ":3000")
for {
conn, _ := listener.Accept()
conn.Write([]byte("hellon"))
conn.Close()
}
}

11. TCP Server With Handler
func main() {
listener, _ := net.Listen("tcp", ":3000")
for {
conn, _ := listener.Accept()
handleConn(conn)
}
}
func handleConn(conn net.Conn) {
conn.Write([]byte("hellon"))
}

12. TCP Server with Goroutine
func main() {
listener, _ := net.Listen("tcp", ":3000")
for {
conn, _ := listener.Accept()
go handleConn(conn)
}
}
func handleConn(conn net.Conn) {
conn.Write([]byte("hellon"))
}

13. []byte
handle octet stream in go

14. binary format (example)
R
Length(14)
R
Reserved(8)
Type(8)
Flags(8)
Stream Identifier(31)
Setting Identifier(24)
Value(32)
[]byte{0, 8, 4, 0, 0, 0, 0, 1, 0, 0,
0, 1, 0, 0, 16, 0}

15. low level byte slice handling
func main() {
b := []byte{}
b = append(b, []byte{0,
b = append(b, 4)
b = append(b, 0)
// stream id 1
b = append(b, []byte{0,
// settings id 1
b = append(b, []byte{0,
// value 4096
b = append(b, []byte{0,
fmt.Println(b)
// [0 8 4 0 0 0 0 1 0 0
}
8}...)
0, 0, 1}...)
0, 0, 1}...)
0, 16, 0}...)
0 1 0 0 16 0]
// length 8
// type 4
// flags 0

16. SWrap (slice wrap)
import “github.com/Jxck/swrap”
type SWrap
●
●
●
●
●
●
●
●
●
●
●
●
func
func
func
func
func
func
func
func
func
func
func
func
New(a []byte) SWrap
(sw *SWrap) Add(a byte)
(sw *SWrap) Bytes() []byte
(sw *SWrap) Compare(b []byte) bool
(sw *SWrap) Delete(i int)
(sw *SWrap) Len() int
(sw *SWrap) Merge(a []byte)
(sw *SWrap) Pop() byte
(sw *SWrap) Push(b byte)
(sw *SWrap) Replace(i int, b byte)
(sw *SWrap) Shift(b byte)
(sw *SWrap) UnShift() byte

17. import “io”
handle IO in go

18. important: io.Reader, io.Writer
type Reader interface {
Read(p []byte) (n int, err error)
}
type Writer interface {
Write(p []byte) (n int, err error)
}
type ReadWriter interface {
Reader
Writer
}

19. net.Conn implements io.Writer
func handleConn(conn net.Conn) {
conn.Write([]byte("hellon"))
}
func handleConn(conn io.Writer) {
conn.Write(
[]byte{104, 101, 108, 108, 111, 10}
)
}

20. import “bytes”
utility for byte slice and Buffer

21. bytes.Buffer: io.ReadWriter for []byte
func handleConn(conn io.Writer) {
conn.Write([]byte{104, 101, 108, 108, 111, 10})
}
func main() {
b := []byte{}
buf := bytes.NewBuffer(b) // io.ReadeWriter
handleConn(buf)
actual := buf.Bytes()
bytes.Equal(
actual,
[]byte{104, 101, 108, 108, 111, 10},
) // true
}

22. import
“encoding/binary”
read/write fixed size value

23. we did...
func main() {
b := []byte{}
b = append(b, []byte{0,
b = append(b, 4)
b = append(b, 0)
// stream id 1
b = append(b, []byte{0,
// settings id 1
b = append(b, []byte{0,
// value 4096
b = append(b, []byte{0,
fmt.Println(b)
// [0 8 4 0 0 0 0 1 0 0
}
8}...)
0, 0, 1}...)
0, 0, 1}...)
0, 16, 0}...)
0 1 0 0 16 0]
// length 8
// type 4
// flags 0

24. Fixed Size
R
Length(14)
R
Type(8)
Flags(8)
Stream Identifier(31)
Reserved(8)
Setting Identifier(24)
Value(32)
var
var
var
var
Length uint16 = 8
Type, Flags uint8 = 4, 0
StreamId, SettingsId uint32 = 1, 1
Value uint32 = 4096

25. encoding/binary.Write()
buf
var
var
var
var
:= bytes.NewBuffer([]byte{})
Length uint16 = 8
Type, Flags uint8 = 4, 0
StreamId, SettingsId uint32 = 1, 1
Value uint32 = 4096
binary.Write(buf, binary.BigEndian, Length)
binary.Write(buf, binary.BigEndian, Type)
// ...
binary.Write(buf, binary.BigEndian, Value)
fmt.Println(buf.Bytes())
// [0 8 4 0 0 0 0 1 0 0 0 1 0 0 16 0]

26. encoding/binary.Write()
buf := bytes.NewBuffer([]byte{})
frame := Frame{
Length:
Type:
Flags:
StreamId:
SettingsId:
Value:
}
8,
4,
0,
1,
1,
4096,
type Frame struct {
Length
uint16
Type
uint8
Flags
uint8
StreamId
uint32
SettingsId uint32
Value
uint32
}
// func Write(w io.Writer, order ByteOrder, data
interface{}) error
binary.Write(buf, binary.BigEndian, frame)
fmt.Println(buf.Bytes())

27. import “net”
tcp client in go

28. TCP Client
func main() {
conn, _ := net.Dial("tcp", ":3000")
b := make([]byte, 100)
n, _ := conn.Read(b)
fmt.Print(string(b[:n]))
// hello
}

29. import “bufio”
buffered io

30. bufio := []byte(4096) + io
// bufio.Reader
type Reader struct {}
// bufio.Writer
type Writer struct {}
func NewReader(rd io.Reader) *Reader {
// convert io.Reader to *bufio.Reader
}
func NewWriter(wr io.Writer) *Writer {
// convert io.Writer to *bufio.Writer
}

31. TCP Client with bufio
func main() {
conn, _ := net.Dial("tcp", ":3000")
br := bufio.NewReader(conn)
line, _ := br.ReadString('n')
fmt.Print(line) // hello
}

32. TCP Client with encoding/binary
type Frame struct {
Length
uint16
Type
uint8
Flags
uint8
StreamId
uint32
SettingsId uint32
Value
uint32
}
func main() {
conn, _ := net.Dial("tcp", ":3000")
frame := &Frame{}
binary.Read(conn, binary.BigEndian, frame)
fmt.Print(frame) // &{8 4 0 1 1 4096}
}

33. HTTP/2.0?

34. HTTP/2.0 with Go
● tcp/tls connection
○ net
● multiplexed stream
○ goroutine / channel
● binary frame
○ static type / encoding/binary
● crypto
○ crypto
● build
○ go build (cross compile)
● test
○ go test (and testing)

35. HTTP2.0 Study (#http2study)
● http2.0 勉強会 #2
○ http://bit.ly/158zE4C
● http2.0 hackathon
○ 12月 or 1月

36. Q & A
anyone ?

37. END
thanks :)