From 41ff983b73b91b31d74cbf7ef03f84e36e432e95 Mon Sep 17 00:00:00 2001 From: liwei Date: Fri, 17 May 2019 19:07:10 +0800 Subject: [PATCH] =?UTF-8?q?BitTorrent=20HTTP=E3=80=81UDP=20tracker=20impl(?= =?UTF-8?q?50%)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- .vscode/settings.json | 18 +- cmd/bt.go | 21 ++ cmd/test.go | 9 + cmd/udpclient.go | 21 ++ cmd/udpserver.go | 27 +++ down/bt/bittorrent.go | 1 + down/bt/metainfo.go | 95 +++++++++ down/bt/tracker.go | 379 +++++++++++++++++++++++++++++++++++ down/{ => http}/down.go | 15 +- down/{ => http}/down_test.go | 13 +- down/{ => http}/request.go | 2 +- down/{ => http}/response.go | 2 +- go.mod | 5 + go.sum | 7 + main.go | 6 +- 16 files changed, 600 insertions(+), 25 deletions(-) create mode 100644 cmd/bt.go create mode 100644 cmd/test.go create mode 100644 cmd/udpclient.go create mode 100644 cmd/udpserver.go create mode 100644 down/bt/bittorrent.go create mode 100644 down/bt/metainfo.go create mode 100644 down/bt/tracker.go rename down/{ => http}/down.go (95%) rename down/{ => http}/down_test.go (91%) rename down/{ => http}/request.go (88%) rename down/{ => http}/response.go (83%) diff --git a/.gitignore b/.gitignore index 1746c97c..9f871611 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -debug \ No newline at end of file +debug + +.idea/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 5d3f8f78..ed7427c7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,17 @@ { - "go.formatTool": "goimports", "go.useLanguageServer": true, - "go.alternateTools": { - "go-langserver": "gopls" - }, "go.languageServerExperimentalFeatures": { - "format": false, - "autoComplete": true + "diagnostics": true // for diagnostics as you type + }, + "[go]": { + "editor.snippetSuggestions": "none", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "gopls": { + "usePlaceholders": true, // add parameter placeholders when completing a function + "enhancedHover": true // experimental to improve quality of hover (will be on by default soon) } } diff --git a/cmd/bt.go b/cmd/bt.go new file mode 100644 index 00000000..c63f6d61 --- /dev/null +++ b/cmd/bt.go @@ -0,0 +1,21 @@ +package main + +import ( + "gopeed/down/bt" +) + +func main() { + metaInfo, _ := bt.ParseFromFile("e:/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent") + // metaInfo, _ := down.ParseFromFile("e:/test.torrent") + if len(metaInfo.AnnounceList) > 0 { + for _, announceArr := range metaInfo.AnnounceList { + if len(announceArr) > 0 { + for _, announce := range announceArr { + bt.DoTracker(announce, metaInfo) + } + } + } + } else { + bt.DoTracker(metaInfo.Announce, metaInfo) + } +} diff --git a/cmd/test.go b/cmd/test.go new file mode 100644 index 00000000..84a67158 --- /dev/null +++ b/cmd/test.go @@ -0,0 +1,9 @@ +package main + +import "fmt" + +func main() { + var b = -1 + var n = uint32(b) + fmt.Println(n) +} diff --git a/cmd/udpclient.go b/cmd/udpclient.go new file mode 100644 index 00000000..63f074ae --- /dev/null +++ b/cmd/udpclient.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "net" +) + +func main() { + ip := net.ParseIP("127.0.0.1") + srcAddr := &net.UDPAddr{IP: net.IPv4zero, Port: 0} + dstAddr := &net.UDPAddr{IP: ip, Port: 9981} + conn, err := net.DialUDP("udp", srcAddr, dstAddr) + if err != nil { + fmt.Println(err) + } + defer conn.Close() + conn.Write([]byte("hello")) + conn.Write([]byte("test")) + conn.Read([]byte{}) + fmt.Printf("<%s>\n", conn.RemoteAddr()) +} diff --git a/cmd/udpserver.go b/cmd/udpserver.go new file mode 100644 index 00000000..28682959 --- /dev/null +++ b/cmd/udpserver.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "net" +) + +func main() { + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 9981}) + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("Local: <%s> \n", conn.LocalAddr().String()) + data := make([]byte, 1024) + for { + n, remoteAddr, err := conn.ReadFromUDP(data) + if err != nil { + fmt.Printf("error during read: %s", err) + } + fmt.Printf("<%s> %s\n", remoteAddr, data[:n]) + _, err = conn.WriteToUDP([]byte("world"), remoteAddr) + if err != nil { + fmt.Printf(err.Error()) + } + } +} diff --git a/down/bt/bittorrent.go b/down/bt/bittorrent.go new file mode 100644 index 00000000..96f76e27 --- /dev/null +++ b/down/bt/bittorrent.go @@ -0,0 +1 @@ +package bt diff --git a/down/bt/metainfo.go b/down/bt/metainfo.go new file mode 100644 index 00000000..895faba8 --- /dev/null +++ b/down/bt/metainfo.go @@ -0,0 +1,95 @@ +package bt + +import ( + "crypto/sha1" + "encoding/json" + "github.com/marksamman/bencode" + "os" +) + +type MetaInfo struct { + // tracker URL + // http://bittorrent.org/beps/bep_0003.html#metainfo-files + Announce string `json:"announce"` + + // tracker URL + // http://bittorrent.org/beps/bep_0012.html + AnnounceList [][]string `json:"announce-list"` + + Comment string `json:"comment"` + + CreatedBy string `json:"created by"` + + CreationDate int64 `json:"creation date"` + + Encoding string `json:"encoding"` + + // http://bittorrent.org/beps/bep_0019.html + UrlList []string `json:"url-list"` + + Info *Info `json:"info"` + InfoHash [20]byte `json:"-"` +} + +type Info struct { + Name string `json:"name"` + PieceLength uint64 `json:"piece length"` + Pieces []string + Length uint64 `json:"length"` + Files []File `json:"files"` +} + +type File struct { + Length uint64 `json:"length"` + Path string `json:"path"` +} + +func (metaInfo *MetaInfo) GetTotalSize() uint64 { + if metaInfo.Info != nil { + if len(metaInfo.Info.Files) > 0 { + var length uint64 + for _, f := range metaInfo.Info.Files { + length += f.Length + } + return length + } else { + return metaInfo.Info.Length + } + } + return 0 +} + +func ParseFromFile(path string) (*MetaInfo, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + // http://bittorrent.org/beps/bep_0003.html#bencoding + btDecode, err := bencode.Decode(file) + if err != nil { + return nil, err + } + bytes, err := json.Marshal(btDecode) + if err != nil { + return nil, err + } + var metaInfo MetaInfo + json.Unmarshal(bytes, &metaInfo) + if metaInfo.Info != nil { + // Encode info hash + info := btDecode["info"].(map[string]interface{}) + infoBtEncode := bencode.Encode(info) + metaInfo.InfoHash = sha1.Sum(infoBtEncode[:]) + // Split pieces hash + pieces := info["pieces"].(string) + if pieces != "" { + metaInfo.Info.Pieces = make([]string, len(pieces)/20) + // pieces parse + for i := range metaInfo.Info.Pieces { + begin := i * 20 + metaInfo.Info.Pieces[i] = pieces[begin : begin+20] + } + } + } + return &metaInfo, nil +} diff --git a/down/bt/tracker.go b/down/bt/tracker.go new file mode 100644 index 00000000..b276409b --- /dev/null +++ b/down/bt/tracker.go @@ -0,0 +1,379 @@ +package bt + +import ( + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "io/ioutil" + "math" + mrand "math/rand" + "net" + "net/http" + "net/url" + "strconv" + "time" +) + +const ( + udpConnectRequestMagic = 0x41727101980 + udpConnectTimeout = 15 + udpConnectRetries = 8 + udpActionConnect = 0 + udpActionAnnounce = 1 + udpActionScrape = 2 + udpActionError = 3 +) + +/* + Offset Size Name Value + 0 64-bit integer protocol_id 0x41727101980 // magic constant + 8 32-bit integer action 0 // connect + 12 32-bit integer transaction_id + 16 + Detail see https://www.libtorrent.org/udp_tracker_protocol.html#connecting +*/ +type UdpConnectRequest struct { + ProtocolId uint64 + Action uint32 + TransactionId uint32 +} + +func NewUdpConnectRequest() *UdpConnectRequest { + return &UdpConnectRequest{ + ProtocolId: udpConnectRequestMagic, + Action: udpActionConnect, + TransactionId: mrand.Uint32(), + } +} + +func (req *UdpConnectRequest) Encode() []byte { + buf := make([]byte, 16) + binary.BigEndian.PutUint64(buf[:8], req.ProtocolId) + binary.BigEndian.PutUint32(buf[8:12], req.Action) + binary.BigEndian.PutUint32(buf[12:], req.TransactionId) + return buf +} + +/* + Offset Size Name Value + 0 32-bit integer action 0 // connect + 4 32-bit integer transaction_id + 8 64-bit integer connection_id + 16 +*/ +type UdpConnectResponse struct { + Action uint32 + TransactionId uint32 + ConnectionId uint64 +} + +func NewUdpConnectResponse(buf []byte) *UdpConnectResponse { + return &UdpConnectResponse{ + Action: binary.BigEndian.Uint32(buf[:4]), + TransactionId: binary.BigEndian.Uint32(buf[4:8]), + ConnectionId: binary.BigEndian.Uint64(buf[8:]), + } +} + +/* + Offset Size Name Value + 0 64-bit integer connection_id + 8 32-bit integer action 1 // announce + 12 32-bit integer transaction_id + 16 20-byte string info_hash + 36 20-byte string peer_id + 56 64-bit integer downloaded + 64 64-bit integer left + 72 64-bit integer uploaded + 80 32-bit integer event 0 // 0: none; 1: completed; 2: started; 3: stopped + 84 32-bit integer IP address 0 // default + 88 32-bit integer key + 92 32-bit integer num_want -1 // default + 96 16-bit integer port + 98 + Detail see https://www.libtorrent.org/udp_tracker_protocol.html#announcing +*/ +type UdpAnnounceRequest struct { + ConnectionId uint64 + Action uint32 + TransactionId uint32 + InfoHash [20]byte + PeerID [20]byte + Downloaded uint64 + Left uint64 + Uploaded uint64 + Event uint32 + IP uint32 + Key uint32 + NumWant int32 + Port uint16 +} + +func NewUdpAnnounceRequest(connectionId uint64) *UdpAnnounceRequest { + return &UdpAnnounceRequest{ + ConnectionId: connectionId, + Action: udpActionAnnounce, + TransactionId: mrand.Uint32(), + } +} + +func (req *UdpAnnounceRequest) encode() []byte { + buf := make([]byte, 98, 98) + binary.BigEndian.PutUint64(buf[:8], req.ConnectionId) + binary.BigEndian.PutUint32(buf[8:12], req.Action) + binary.BigEndian.PutUint32(buf[12:16], req.TransactionId) + copy(buf[16:36], req.InfoHash[:]) + copy(buf[36:56], req.PeerID[:]) + binary.BigEndian.PutUint64(buf[56:64], req.Downloaded) + binary.BigEndian.PutUint64(buf[64:72], req.Left) + binary.BigEndian.PutUint64(buf[72:80], req.Uploaded) + binary.BigEndian.PutUint32(buf[80:84], req.Event) + binary.BigEndian.PutUint32(buf[84:88], req.IP) + binary.BigEndian.PutUint32(buf[88:92], req.Key) + binary.BigEndian.PutUint32(buf[92:96], uint32(req.NumWant)) + binary.BigEndian.PutUint16(buf[96:98], req.Port) + return buf +} + +/* + Offset Size Name Value + 0 32-bit integer action 1 // announce + 4 32-bit integer transaction_id + 8 32-bit integer interval + 12 32-bit integer leechers + 16 32-bit integer seeders + 20 + 6 * n 32-bit integer IP address + 24 + 6 * n 16-bit integer TCP port + 20 + 6 * N +*/ +type UdpAnnounceResponse struct { + Action uint32 + TransactionId uint32 + Interval uint32 + Leechers uint32 + Seeders uint32 + IP []uint32 + Port []uint16 +} + +func NewUdpAnnounceResponse(buf []byte) *UdpAnnounceResponse { + response := &UdpAnnounceResponse{ + Action: binary.BigEndian.Uint32(buf[:4]), + TransactionId: binary.BigEndian.Uint32(buf[4:8]), + Interval: binary.BigEndian.Uint32(buf[8:12]), + Leechers: binary.BigEndian.Uint32(buf[12:16]), + Seeders: binary.BigEndian.Uint32(buf[16:20]), + } + count := (len(buf) - 20) / 6 + response.IP = make([]uint32, count, count) + response.Port = make([]uint16, count, count) + for i := 0; i < count; i++ { + begin := 20 + 6*i + response.IP[i] = binary.BigEndian.Uint32(buf[begin : begin+4]) + response.Port[i] = binary.BigEndian.Uint16(buf[begin+4 : begin+6]) + } + return response +} + +type UdpTrackerContent struct { + URL *url.URL + MetaInfo *MetaInfo + Conn *net.UDPConn + ConnectRequest *UdpConnectRequest + ConnectResponse *UdpConnectResponse + AnnounceRequest *UdpAnnounceRequest + AnnounceResponse *UdpAnnounceResponse +} + +func NewUdpTrackerContent(url *url.URL, metaInfo *MetaInfo) *UdpTrackerContent { + return &UdpTrackerContent{ + URL: url, + MetaInfo: metaInfo, + } +} + +func (content *UdpTrackerContent) connect() error { + host, portStr, err := net.SplitHostPort(content.URL.Host) + if err != nil { + fmt.Println(err) + return err + } + port, err := strconv.ParseInt(portStr, 10, 32) + if err != nil { + fmt.Println(err) + return err + } + ip, err := lookupIP(host, content.URL.Scheme) + if err != nil { + fmt.Println(err) + return err + } + + srcAddr := &net.UDPAddr{IP: net.IPv4zero, Port: 0} + dstAddr := &net.UDPAddr{ + IP: ip, + Port: int(port), + } + conn, err := net.DialUDP(content.URL.Scheme, srcAddr, dstAddr) + if err != nil { + fmt.Println(err) + return err + } + // Connect + request := NewUdpConnectRequest() + var response *UdpConnectResponse + for n := 0; n < udpConnectRetries; n++ { + buf := request.Encode() + conn.Write(buf) + // 15 * 2 ^ n seconds + timeout := udpConnectTimeout * math.Pow(2, float64(n)) + conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(timeout))) + len, err := conn.Read(buf) + if len == 16 && err == nil { + response = NewUdpConnectResponse(buf) + break + } + } + if response == nil { + conn.Close() + return errors.New("UDP tracker connect error") + } + if response.Action != udpActionConnect || request.TransactionId != response.TransactionId { + conn.Close() + return errors.New("UDP tracker connect response error") + } + content.Conn = conn + content.ConnectRequest = request + content.ConnectResponse = response + return nil +} + +func (content *UdpTrackerContent) announce() error { + request := NewUdpAnnounceRequest(content.ConnectResponse.ConnectionId) + request.InfoHash = content.MetaInfo.InfoHash + peerID, err := genPeerID() + if err != nil { + fmt.Println(err) + return err + } + request.PeerID = peerID + request.Downloaded = 0 + request.Uploaded = 0 + request.Left = content.MetaInfo.GetTotalSize() - request.Downloaded + request.Event = 0 + request.IP = 0 + request.Key = 0 + request.NumWant = 50 + request.Port = 6882 + encode := request.encode() + content.Conn.Write(encode) + buf := make([]byte, 512) + len, err := content.Conn.Read(buf) + if err != nil { + fmt.Println(err) + return err + } + response := NewUdpAnnounceResponse(buf[:len]) + fmt.Println(response) + return nil +} + +func DoTracker(announce string, metaInfo *MetaInfo) { + if announce != "" { + url, _ := url.Parse(announce) + switch url.Scheme { + case "http", "https": + // TODO UDP test + break + httpTracker(url, metaInfo) + case "udp", "udp4", "udp6": + udpTracker(url, metaInfo) + } + } +} + +func genPeerID() ([20]byte, error) { + peerId := [20]byte{'G', 'P', '0', '0', '1'} + _, err := rand.Read(peerId[5:]) + if err != nil { + return peerId, err + } + return peerId, nil +} + +// http://bittorrent.org/beps/bep_0003.html#trackers +func httpTracker(url *url.URL, metaInfo *MetaInfo) { + query := url.Query() + query.Add("info_hash", string(metaInfo.InfoHash[:])) + // Generate peer ID + peerID, err := genPeerID() + if err != nil { + fmt.Println(err) + return + } + query.Add("peer_id", string(peerID[:])) + query.Add("port", "6882") + query.Add("uploaded", "0") + query.Add("downloaded", "0") + query.Add("left", strconv.FormatInt(int64(metaInfo.GetTotalSize()), 10)) + url.RawQuery = query.Encode() + request, err := http.NewRequest("GET", url.String(), nil) + if err != nil { + fmt.Println(err) + return + } + var httpClient = &http.Client{ + Timeout: time.Second * time.Duration(15), + } + request.Header.Set("User-Agent", "gopeed/0.0.1") + fmt.Println("request:" + url.String()) + response, err := httpClient.Do(request) + if err != nil { + fmt.Println(err) + return + } + defer response.Body.Close() + resp, err := ioutil.ReadAll(response.Body) + if err != nil { + if err != io.EOF { + fmt.Println(err) + return + } + } + fmt.Println(string(resp)) +} + +// http://bittorrent.org/beps/bep_0015.html +func udpTracker(url *url.URL, metaInfo *MetaInfo) { + content := NewUdpTrackerContent(url, metaInfo) + err := content.connect() + if err != nil { + fmt.Println(err) + return + } + content.announce() +} + +func lookupIP(host string, scheme string) (ip net.IP, err error) { + ips, _ := net.LookupIP(host) + if len(ips) == 0 { + err = errors.New("no ips") + return + } + for _, ip = range ips { + switch scheme { + case "udp", "udp4": + if ip.To4() != nil { + return + } + case "udp6": + if ip.To4() == nil { + return + } + } + } + err = errors.New("no available ips ") + return +} diff --git a/down/down.go b/down/http/down.go similarity index 95% rename from down/down.go rename to down/http/down.go index f9ceb2ee..25b21bf0 100644 --- a/down/down.go +++ b/down/http/down.go @@ -1,4 +1,4 @@ -package down +package http import ( "fmt" @@ -13,6 +13,11 @@ import ( "sync" ) +type Download interface { + Resolve(request *http.Request) (*http.Response, error) + Down(request *http.Request) error +} + // Resolve return the file response to be downloaded func Resolve(request *Request) (*Response, error) { httpRequest, err := BuildHTTPRequest(request) @@ -144,11 +149,9 @@ func BuildHTTPClient() *http.Client { } func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup, fileLock *sync.Mutex) { - defer func() { - if waitGroup != nil { - waitGroup.Done() - } - }() + if waitGroup != nil { + defer waitGroup.Done() + } httpRequest, _ := BuildHTTPRequest(request) httpRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end)) fmt.Printf("down %d-%d\n", start, end) diff --git a/down/down_test.go b/down/http/down_test.go similarity index 91% rename from down/down_test.go rename to down/http/down_test.go index b5169018..ea8c11e8 100644 --- a/down/down_test.go +++ b/down/http/down_test.go @@ -1,4 +1,4 @@ -package down +package http import ( "crypto/md5" @@ -131,20 +131,19 @@ func TestTemp(t *testing.T) { func fileMd5(filePath string) string { file, _ := os.Open(filePath) - //Tell the program to call the following function when the current function returns + // Tell the program to call the following function when the current function returns defer file.Close() - //Open a new hash interface to write to + // Open a new hash interface to write to hash := md5.New() - //Copy the file in the hash interface and check for any error + // Copy the file in the hash interface and check for any error if _, err := io.Copy(hash, file); err != nil { return "" } return hex.EncodeToString(hash.Sum(nil)) } -func TestMd5(t *testing.T) { - //8de7a6a2e786861013d61b77b2394012 - fmt.Println(fileMd5("d:/favicon.ico")) +func TestTemp2(t *testing.T) { + fmt.Println("123456789"[2:3]) } diff --git a/down/request.go b/down/http/request.go similarity index 88% rename from down/request.go rename to down/http/request.go index 98fc3042..45b11372 100644 --- a/down/request.go +++ b/down/http/request.go @@ -1,4 +1,4 @@ -package down +package http type Request struct { Method string diff --git a/down/response.go b/down/http/response.go similarity index 83% rename from down/response.go rename to down/http/response.go index 5306f76d..4bfe9949 100644 --- a/down/response.go +++ b/down/http/response.go @@ -1,4 +1,4 @@ -package down +package http type Response struct { Name string diff --git a/go.mod b/go.mod index fbced7e4..8e78fd94 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module gopeed go 1.12 + +require ( + github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e + golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 +) diff --git a/go.sum b/go.sum index e69de29b..a5c4caa9 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,7 @@ +github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e h1:KMs6SK8iDSR1+ZzOK10L5wGPpWDByyvOe5nrqk51g2U= +github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e/go.mod h1:+AHfJo5+69p+fjvMJTmYajNP9rFBHQcaTDFmuXRRATI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index ee61e4b2..3c93d228 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,11 @@ package main import ( "fmt" - "gopeed/down" + "gopeed/down/http" ) func main() { - request := &down.Request{ + request := &http.Request{ Method: "get", URL: "http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar", Header: map[string]string{ @@ -18,7 +18,7 @@ func main() { "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", }, } - got, _ := down.Resolve(request) + got, _ := http.Resolve(request) fmt.Println(got) // webview.Open("Minimal webview example", "https://www.baidu.com", 800, 600, true) }