From 6eadbc3c6b3a6715aa43e1f5987841b736516fcd Mon Sep 17 00:00:00 2001 From: ren yuze Date: Tue, 21 Jul 2020 09:20:02 +0800 Subject: [PATCH] add curl with file --- curl.go | 197 +++++++++++++++++++++++++-------- http.go | 338 -------------------------------------------------------- 2 files changed, 151 insertions(+), 384 deletions(-) delete mode 100644 http.go diff --git a/curl.go b/curl.go index bab0f3e..7481519 100644 --- a/curl.go +++ b/curl.go @@ -6,39 +6,50 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" + "os" "time" ) +type RequestFile struct { + UploadFile string + UploadForm map[string]string + UploadName string +} type Request struct { - TimeOut int - DialTimeOut int - Url string - Method string - RecvData []byte - WriteRecvData bool - RecvIo io.Writer - ReqHeader http.Header - ReqCookies []*http.Cookie - RespHeader http.Header - RespCookies []*http.Cookie - RespHttpCode int - PostBuffer *bytes.Buffer - CircleBuffer *CircleByteBuffer - Proxy string + TimeOut int + DialTimeOut int + Url string + Method string + RecvData []byte + RecvContentLength int64 + WriteRecvData bool + RecvIo io.Writer + ReqHeader http.Header + ReqCookies []*http.Cookie + RespHeader http.Header + RespCookies []*http.Cookie + RequestFile + RespHttpCode int + PostBuffer *bytes.Buffer + CircleBuffer *CircleByteBuffer + Proxy string + Process func(float64) } func NewRequests(url string, postdata []byte, method string) Request { - return Request{ - TimeOut: 30, - DialTimeOut: 15, - Url: url, - PostBuffer: bytes.NewBuffer(postdata), - Method: method, + req := Request{ + TimeOut: 30, + DialTimeOut: 15, + Url: url, + PostBuffer: bytes.NewBuffer(postdata), + Method: method, + WriteRecvData: true, } + req.ReqHeader = make(http.Header) + return req } func (curl *Request) ResetReqHeader() { @@ -63,17 +74,130 @@ func randomBoundary() string { } func Curl(curl Request) (resps Request, err error) { + var fpsrc *os.File + if curl.RequestFile.UploadFile != "" { + fpsrc, err = os.Open(curl.UploadFile) + if err != nil { + return + } + defer fpsrc.Close() + boundary := randomBoundary() + boundarybytes := []byte("\r\n--" + boundary + "\r\n") + endbytes := []byte("\r\n--" + boundary + "--\r\n") + fpstat, _ := fpsrc.Stat() + filebig := float64(fpstat.Size()) + sum, n := 0, 0 + fpdst := NewCircleByteBuffer(1048576) + if curl.UploadForm != nil { + for k, v := range curl.UploadForm { + header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\";\r\nContent-Type: x-www-form-urlencoded \r\n\r\n", k) + fpdst.Write(boundarybytes) + fpdst.Write([]byte(header)) + fpdst.Write([]byte(v)) + } + } + header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\nContent-Type: application/octet-stream\r\n\r\n", curl.UploadName, fpstat.Name()) + fpdst.Write(boundarybytes) + fpdst.Write([]byte(header)) + go func() { + for { + bufs := make([]byte, 393213) + n, err = fpsrc.Read(bufs) + if err != nil { + if err == io.EOF { + if n != 0 { + fpdst.Write(bufs[0:n]) + if curl.Process != nil { + go curl.Process(float64(sum+n) / filebig * 100) + } + } + break + } + return + } + sum += n + if curl.Process != nil { + go curl.Process(float64(sum+n) / filebig * 100) + } + fpdst.Write(bufs[0:n]) + } + fpdst.Write(endbytes) + fpdst.Write(nil) + }() + curl.CircleBuffer = fpdst + curl.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+boundary) + } + resp, err := netcurl(curl) + if err != nil { + return Request{}, err + } + defer resp.Body.Close() + curl.PostBuffer = nil + curl.CircleBuffer = nil + curl.RespHttpCode = resp.StatusCode + curl.RespHeader = resp.Header + curl.RespCookies = resp.Cookies() + curl.RecvContentLength = resp.ContentLength + readFunc := func(reader io.ReadCloser, writer io.Writer) error { + lengthall := resp.ContentLength + defer reader.Close() + var lengthsum int + buf := make([]byte, 65535) + for { + n, err := reader.Read(buf) + if n != 0 { + _, err := writer.Write(buf[:n]) + lengthsum += n + if curl.Process != nil { + go curl.Process(float64(lengthsum) / float64(lengthall) * 100.00) + } + if err != nil { + return err + } + } + if err != nil && err != io.EOF { + return err + } else if err == io.EOF { + return nil + } + } + } + if curl.WriteRecvData { + buf := bytes.NewBuffer([]byte{}) + err = readFunc(resp.Body, buf) + if err != nil { + return + } + curl.RecvData = buf.Bytes() + } + if curl.RecvIo != nil { + if curl.WriteRecvData { + _, err = curl.RecvIo.Write(curl.RecvData) + } else { + err = readFunc(resp.Body, curl.RecvIo) + if err != nil { + return + } + } + } + return curl, err +} + +func netcurl(curl Request) (*http.Response, error) { var req *http.Request + var err error if curl.Method == "" { - return Request{}, errors.New("Error Method Not Entered") + return nil, errors.New("Error Method Not Entered") } - if curl.PostBuffer != nil { + if curl.PostBuffer != nil && curl.PostBuffer.Len() > 0 { req, err = http.NewRequest(curl.Method, curl.Url, curl.PostBuffer) - } else { + } else if curl.CircleBuffer != nil && curl.CircleBuffer.getLen() > 0 { req, err = http.NewRequest(curl.Method, curl.Url, curl.CircleBuffer) + } else { + req, err = http.NewRequest(curl.Method, curl.Url, nil) } if err != nil { - return + return nil, err } req.Header = curl.ReqHeader if len(curl.ReqCookies) != 0 { @@ -102,24 +226,5 @@ func Curl(curl Request) (resps Request, err error) { Transport: transport, } resp, err := client.Do(req) - if err != nil { - return - } - defer resp.Body.Close() - curl.PostBuffer = nil - curl.CircleBuffer = nil - curl.RespHttpCode = resp.StatusCode - curl.RespHeader = resp.Header - curl.RespCookies = resp.Cookies() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - if curl.WriteRecvData { - curl.RecvData = body - } - if curl.RecvIo != nil { - _, err = curl.RecvIo.Write(body) - } - return curl, err + return resp, err } diff --git a/http.go b/http.go deleted file mode 100644 index 6041fef..0000000 --- a/http.go +++ /dev/null @@ -1,338 +0,0 @@ -package starnet - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "net" - "net/http" - urls "net/url" - "os" - "path/filepath" - "time" -) - -type StarCurl struct { - TimeOut int - DialTimeOut int - ReqHeader http.Header - ReqCookies []*http.Cookie - RespHeader http.Header - RespCookies []*http.Cookie - RespHttpCode int - PostBuffer *bytes.Buffer - CircleBuffer *CircleByteBuffer - Proxy string -} - -func NewStarCurl() *StarCurl { - star := new(StarCurl) - star.ReqHeader = make(http.Header) - star.TimeOut = 60 - star.DialTimeOut = 15 - star.PostBuffer = nil - return star -} - -func (this *StarCurl) ResetReqHeader() { - this.ReqHeader = make(http.Header) -} - -func (this *StarCurl) ResetReqCookies() { - this.ReqCookies = []*http.Cookie{} -} - -func (this *StarCurl) AddSimpleCookie(key, value string) { - this.ReqCookies = append(this.ReqCookies, &http.Cookie{Name: key, Value: value, Path: "/"}) -} - -func (this *StarCurl) CurlWithFile(url string, postdata map[string]string, formname, fpath, savepath string, tofile bool, shell func(float64)) (result []byte, err error) { - fpsrc, err := os.Open(fpath) - if err != nil { - return - } - defer fpsrc.Close() - boundary := randomBoundary() - boundarybytes := []byte("\r\n--" + boundary + "\r\n") - endbytes := []byte("\r\n--" + boundary + "--\r\n") - fpstat, _ := os.Stat(fpath) - filebig := float64(fpstat.Size()) - sum, n := 0, 0 - fpdst := NewCircleByteBuffer(1048576) - if postdata != nil { - for k, v := range postdata { - header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\";\r\nContent-Type: x-www-form-urlencoded \r\n\r\n", k) - fpdst.Write(boundarybytes) - fpdst.Write([]byte(header)) - fpdst.Write([]byte(v)) - } - } - header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\nContent-Type: application/octet-stream\r\n\r\n", formname, fpstat.Name()) - fpdst.Write(boundarybytes) - fpdst.Write([]byte(header)) - go func() { - for { - bufs := make([]byte, 393213) - n, err = fpsrc.Read(bufs) - if err != nil { - if err == io.EOF { - if n != 0 { - fpdst.Write(bufs[0:n]) - go shell(float64(sum+n) / filebig * 100) - } - break - } - return - } - sum += n - go shell(float64(sum) / filebig * 100) - fpdst.Write(bufs[0:n]) - } - fpdst.Write(endbytes) - fpdst.Write(nil) - }() - this.CircleBuffer = fpdst - this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+boundary) - if tofile { - err = this.CurlDataToFile(url, []byte{}, "POST", savepath, shell) - this.ResetReqHeader() - } else { - result, err = this.Curl(url, []byte{}, "POST") - } - this.ResetReqHeader() - return -} - -func (this *StarCurl) CurlWithFileByBytes(url string, postdata map[string]string, formname, fname string, data []byte, savepath string, tofile bool) (result []byte, err error) { - buf := &bytes.Buffer{} - bufwriter := multipart.NewWriter(buf) - if postdata != nil { - for k, v := range postdata { - bufwriter.WriteField(k, v) - } - } - fpdst, err := bufwriter.CreateFormFile(formname, fname) - if err != nil { - return - } - fpdst.Write(data) - this.PostBuffer = buf - this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+bufwriter.Boundary()) - bufwriter.Close() - if tofile { - err = this.CurlDataToFile(url, []byte{}, "POST", savepath, func(float64) {}) - this.ResetReqHeader() - } else { - result, err = this.Curl(url, []byte{}, "POST") - } - this.ResetReqHeader() - return -} - -func (this *StarCurl) CurlWithFileByMemory(url string, postdata map[string]string, formname, fpath, savepath string, tofile bool, shell func(float64)) (result []byte, err error) { - buf := &bytes.Buffer{} - bufwriter := multipart.NewWriter(buf) - if postdata != nil { - for k, v := range postdata { - bufwriter.WriteField(k, v) - } - } - fpdst, err := bufwriter.CreateFormFile(formname, filepath.Base(fpath)) - if err != nil { - return - } - fpsrc, err := os.Open(fpath) - if err != nil { - return - } - defer fpsrc.Close() - fpstat, _ := os.Stat(fpath) - filebig := float64(fpstat.Size()) - sum, n := 0, 0 - for { - bufs := make([]byte, 393213) - n, err = fpsrc.Read(bufs) - if err != nil { - if err == io.EOF { - if n != 0 { - fpdst.Write(bufs[0:n]) - go shell(float64(sum+n) / filebig * 100) - } - break - } - return - } - sum += n - go shell(float64(sum) / filebig * 100) - fpdst.Write(bufs[0:n]) - } - - this.PostBuffer = buf - this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+bufwriter.Boundary()) - bufwriter.Close() - if tofile { - err = this.CurlDataToFile(url, []byte{}, "POST", savepath, shell) - this.ResetReqHeader() - } else { - result, err = this.Curl(url, []byte{}, "POST") - } - this.ResetReqHeader() - return -} - -func (this *StarCurl) CurlDataToFile(url string, postdata []byte, method, fpath string, shell func(float64)) (err error) { - var req *http.Request - if method == "" { - if len(postdata) != 0 { - method = "POST" - } else { - method = "GET" - } - } - if len(postdata) == 0 && this.PostBuffer == nil && this.CircleBuffer == nil { - req, err = http.NewRequest(method, url, nil) - } else if len(postdata) != 0 { - req, err = http.NewRequest(method, url, bytes.NewBuffer(postdata)) - } else if this.PostBuffer != nil { - req, err = http.NewRequest(method, url, this.PostBuffer) - } else { - req, err = http.NewRequest(method, url, this.CircleBuffer) - } - if err != nil { - return - } - if (this.ReqHeader == nil) && method == "POST" { - this.ReqHeader.Set("Content-Type", "application/x-www-form-urlencoded") - } - req.Header = this.ReqHeader - if len(this.ReqCookies) != 0 { - for _, v := range this.ReqCookies { - req.AddCookie(v) - } - } - transport := &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - deadline := time.Now().Add(time.Duration(this.TimeOut) * time.Second) - c, err := net.DialTimeout(netw, addr, time.Second*time.Duration(this.DialTimeOut)) - if err != nil { - return nil, err - } - if this.TimeOut != 0 { - c.SetDeadline(deadline) - } - return c, nil - }, - } - if this.Proxy != "" { - purl, _ := urls.Parse(this.Proxy) - transport.Proxy = http.ProxyURL(purl) - } - client := &http.Client{ - Transport: transport, - } - resp, err := client.Do(req) - if err != nil { - return - } - defer resp.Body.Close() - this.PostBuffer = nil - this.CircleBuffer = nil - this.RespHttpCode = resp.StatusCode - this.RespHeader = resp.Header - this.RespCookies = resp.Cookies() - fpsrc, err := os.Create(fpath) - if err != nil { - return - } - defer fpsrc.Close() - filebig := float64(resp.ContentLength) - if filebig <= 0 { - filebig = 100 - } - var n, sum int = 0, 0 - for { - buf := make([]byte, 393213) - n, err = resp.Body.Read(buf) - if err != nil { - if err == io.EOF { - err = nil - if n != 0 { - fpsrc.Write(buf[0:n]) - } - go shell(100.00) - break - } - return - } - sum += n - go shell(float64(sum) / filebig * 100.00) - fpsrc.Write(buf[0:n]) - } - return -} - -func (this *StarCurl) Curl(url string, postdata []byte, method string) (body []byte, err error) { - var req *http.Request - if method == "" { - if len(postdata) != 0 { - method = "POST" - } else { - method = "GET" - } - } - if len(postdata) == 0 && this.PostBuffer == nil && this.CircleBuffer == nil { - req, err = http.NewRequest(method, url, nil) - } else if len(postdata) != 0 { - req, err = http.NewRequest(method, url, bytes.NewBuffer(postdata)) - } else if this.PostBuffer != nil { - req, err = http.NewRequest(method, url, this.PostBuffer) - } else { - req, err = http.NewRequest(method, url, this.CircleBuffer) - } - if err != nil { - return - } - if (this.ReqHeader == nil) && method == "POST" { - this.ReqHeader.Set("Content-Type", "application/x-www-form-urlencoded") - } - req.Header = this.ReqHeader - if len(this.ReqCookies) != 0 { - for _, v := range this.ReqCookies { - req.AddCookie(v) - } - } - transport := &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - deadline := time.Now().Add(time.Duration(this.TimeOut) * time.Second) - c, err := net.DialTimeout(netw, addr, time.Second*time.Duration(this.DialTimeOut)) - if err != nil { - return nil, err - } - if this.TimeOut != 0 { - c.SetDeadline(deadline) - } - return c, nil - }, - } - if this.Proxy != "" { - purl, _ := urls.Parse(this.Proxy) - transport.Proxy = http.ProxyURL(purl) - } - client := &http.Client{ - Transport: transport, - } - resp, err := client.Do(req) - if err != nil { - return - } - defer resp.Body.Close() - this.PostBuffer = nil - this.CircleBuffer = nil - this.RespHttpCode = resp.StatusCode - this.RespHeader = resp.Header - this.RespCookies = resp.Cookies() - body, err = ioutil.ReadAll(resp.Body) - return -}