package starnet import ( "bytes" "context" "crypto/tls" "encoding/json" "fmt" "io" "mime/multipart" "net" "net/http" "net/url" "os" "strconv" "strings" "time" ) const ( HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded` HEADER_FORM_DATA = `multipart/form-data` HEADER_JSON = `application/json` HEADER_PLAIN = `text/plain` ) var ( DefaultDialTimeout = 5 * time.Second DefaultTimeout = 10 * time.Second DefaultFetchRespBody = false ) func UrlEncodeRaw(str string) string { strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1) return strs } func UrlEncode(str string) string { return url.QueryEscape(str) } func UrlDecode(str string) (string, error) { return url.QueryUnescape(str) } func BuildQuery(queryData map[string]string) string { query := url.Values{} for k, v := range queryData { query.Add(k, v) } return query.Encode() } // BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string, // and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests, // where the server expects the request body to be URL-encoded form data. // // Parameters: // queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request. // // Returns: // A byte slice representing the URL-encoded form data. func BuildPostForm(queryMap map[string]string) []byte { return []byte(BuildQuery(queryMap)) } func Get(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "GET", opts...).Do() } func Post(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "POST", opts...).Do() } func Options(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "OPTIONS", opts...).Do() } func Put(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "PUT", opts...).Do() } func Delete(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "DELETE", opts...).Do() } func Head(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "HEAD", opts...).Do() } func Patch(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "PATCH", opts...).Do() } func Trace(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "TRACE", opts...).Do() } func Connect(uri string, opts ...RequestOpt) (*Response, error) { return NewSimpleRequest(uri, "CONNECT", opts...).Do() } type Request struct { ctx context.Context uri string method string errInfo error RequestOpts } func (r *Request) Method() string { return r.method } func (r *Request) SetMethod(method string) error { method = strings.ToUpper(method) if !validMethod(method) { return fmt.Errorf("invalid method: %s", method) } r.method = method r.rawRequest.Method = method return nil } func (r *Request) SetMethodNoError(method string) *Request { r.SetMethod(method) return r } func (r *Request) Uri() string { return r.uri } func (r *Request) SetUri(uri string) error { u, err := url.Parse(uri) if err != nil { return fmt.Errorf("parse uri error: %s", err) } r.uri = uri u.Host = removeEmptyPort(u.Host) r.rawRequest.Host = u.Host r.rawRequest.URL = u return nil } func (r *Request) SetUriNoError(uri string) *Request { r.SetUri(uri) return r } func (r *Request) RawRequest() *http.Request { return r.rawRequest } func (r *Request) SetRawRequest(rawRequest *http.Request) *Request { r.rawRequest = rawRequest return r } func (r *Request) RawClient() *http.Client { return r.rawClient } func (r *Request) SetRawClient(rawClient *http.Client) *Request { r.rawClient = rawClient return r } func (r *Request) Do() (*Response, error) { return Curl(r) } func (r *Request) Get() (*Response, error) { err := r.SetMethod("GET") if err != nil { return nil, err } return Curl(r) } func (r *Request) Post(data []byte) (*Response, error) { err := r.SetMethod("POST") if err != nil { return nil, err } r.bodyDataBytes = data r.bodyDataReader = nil return Curl(r) } type RequestOpts struct { rawRequest *http.Request rawClient *http.Client alreadyApply bool bodyDataBytes []byte bodyDataReader io.Reader bodyFormData map[string][]string bodyFileData []RequestFile //以上优先度为 bodyDataReader> bodyDataBytes > bodyFormData > bodyFileData FileUploadRecallFn func(filename string, upPos int64, total int64) proxy string timeout time.Duration dialTimeout time.Duration headers http.Header cookies []*http.Cookie transport *http.Transport queries map[string][]string disableRedirect bool //doRawRequest=true 不对request修改,直接发送 doRawRequest bool //doRawClient=true 不对http client修改,直接发送 doRawClient bool //doRawTransPort=true 不对http transport修改,直接发送 doRawTransport bool skipTLSVerify bool tlsConfig *tls.Config autoFetchRespBody bool customIP []string alreadySetLookUpIPfn bool lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error) customDNS []string basicAuth [2]string autoCalcContentLength bool } func (r *Request) AutoCalcContentLength() bool { return r.autoCalcContentLength } // SetAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body. // WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding. // also the memory usage will be higher // Note that this function will not work if doRawRequest is true func (r *Request) SetAutoCalcContentLength(autoCalcContentLength bool) *Request { r.autoCalcContentLength = autoCalcContentLength return r } // BasicAuth returns the username and password provided in the request's Authorization header. func (r *RequestOpts) BasicAuth() (string, string) { return r.basicAuth[0], r.basicAuth[1] } // SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password. // Note: If doRawRequest is true, this function will nolonger work func (r *RequestOpts) SetBasicAuth(username, password string) *RequestOpts { r.basicAuth = [2]string{username, password} return r } func (r *Request) DoRawTransport() bool { return r.doRawTransport } func (r *Request) SetDoRawTransport(doRawTransport bool) *Request { r.doRawTransport = doRawTransport return r } func (r *Request) CustomDNS() []string { return r.customDNS } // Note: if LookUpIPfn is set, this function will not be used func (r *Request) SetCustomDNS(customDNS []string) error { for _, v := range customDNS { if net.ParseIP(v) == nil { return fmt.Errorf("invalid custom dns: %s", v) } } r.customDNS = customDNS return nil } // Note: if LookUpIPfn is set, this function will not be used func (r *Request) SetCustomDNSNoError(customDNS []string) *Request { r.SetCustomDNS(customDNS) return r } // Note: if LookUpIPfn is set, this function will not be used func (r *Request) AddCustomDNS(customDNS []string) error { for _, v := range customDNS { if net.ParseIP(v) == nil { return fmt.Errorf("invalid custom dns: %s", v) } } r.customDNS = customDNS return nil } // Note: if LookUpIPfn is set, this function will not be used func (r *Request) AddCustomDNSNoError(customDNS []string) *Request { r.AddCustomDNS(customDNS) return r } func (r *Request) LookUpIPfn() func(ctx context.Context, host string) ([]net.IPAddr, error) { return r.lookUpIPfn } func (r *Request) SetLookUpIPfn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) *Request { if lookUpIPfn == nil { r.alreadySetLookUpIPfn = false r.lookUpIPfn = net.DefaultResolver.LookupIPAddr return r } r.lookUpIPfn = lookUpIPfn r.alreadySetLookUpIPfn = true return r } func (r *Request) CustomHostIP() []string { return r.customIP } func (r *Request) SetCustomHostIP(customIP []string) *Request { r.customIP = customIP return r } func (r *Request) AddCustomHostIP(customIP string) *Request { r.customIP = append(r.customIP, customIP) return r } func (r *Request) BodyDataBytes() []byte { return r.bodyDataBytes } func (r *Request) SetBodyDataBytes(bodyDataBytes []byte) *Request { r.bodyDataBytes = bodyDataBytes return r } func (r *Request) BodyDataReader() io.Reader { return r.bodyDataReader } func (r *Request) SetBodyDataReader(bodyDataReader io.Reader) *Request { r.bodyDataReader = bodyDataReader return r } func (r *Request) BodyFormData() map[string][]string { return r.bodyFormData } func (r *Request) SetBodyFormData(bodyFormData map[string][]string) *Request { r.bodyFormData = bodyFormData return r } func (r *Request) SetFormData(bodyFormData map[string][]string) *Request { return r.SetBodyFormData(bodyFormData) } func (r *Request) AddFormMapData(bodyFormData map[string]string) *Request { for k, v := range bodyFormData { r.bodyFormData[k] = append(r.bodyFormData[k], v) } return r } func (r *Request) AddFormData(k, v string) *Request { r.bodyFormData[k] = append(r.bodyFormData[k], v) return r } func (r *Request) BodyFileData() []RequestFile { return r.bodyFileData } func (r *Request) SetBodyFileData(bodyFileData []RequestFile) *Request { r.bodyFileData = bodyFileData return r } func (r *Request) Proxy() string { return r.proxy } func (r *Request) SetProxy(proxy string) *Request { r.proxy = proxy return r } func (r *Request) Timeout() time.Duration { return r.timeout } func (r *Request) SetTimeout(timeout time.Duration) *Request { r.timeout = timeout return r } func (r *Request) DialTimeout() time.Duration { return r.dialTimeout } func (r *Request) SetDialTimeout(dialTimeout time.Duration) *Request { r.dialTimeout = dialTimeout return r } func (r *Request) Headers() http.Header { return r.headers } func (r *Request) SetHeaders(headers http.Header) *Request { r.headers = headers return r } func (r *Request) AddHeader(key, val string) *Request { r.headers.Add(key, val) return r } func (r *Request) SetHeader(key, val string) *Request { r.headers.Set(key, val) return r } func (r *Request) SetContentType(ct string) *Request { r.headers.Set("Content-Type", ct) return r } func (r *Request) SetUserAgent(ua string) *Request { r.headers.Set("User-Agent", ua) return r } func (r *Request) DeleteHeader(key string) *Request { r.headers.Del(key) return r } func (r *Request) Cookies() []*http.Cookie { return r.cookies } func (r *Request) SetCookies(cookies []*http.Cookie) *Request { r.cookies = cookies return r } func (r *Request) Transport() *http.Transport { return r.transport } func (r *Request) SetTransport(transport *http.Transport) *Request { r.transport = transport return r } func (r *Request) Queries() map[string][]string { return r.queries } func (r *Request) SetQueries(queries map[string][]string) *Request { r.queries = queries return r } func (r *Request) AddQueries(queries map[string]string) *Request { for k, v := range queries { r.queries[k] = append(r.queries[k], v) } return r } func (r *Request) AddQuery(key, value string) *Request { r.queries[key] = append(r.queries[key], value) return r } func (r *Request) DelQueryKv(key, value string) *Request { if _, ok := r.queries[key]; !ok { return r } for i, v := range r.queries[key] { if v == value { r.queries[key] = append(r.queries[key][:i], r.queries[key][i+1:]...) } } return r } func (r *Request) DelQuery(key string) *Request { if _, ok := r.queries[key]; !ok { return r } delete(r.queries, key) return r } func (r *Request) DisableRedirect() bool { return r.disableRedirect } func (r *Request) SetDisableRedirect(disableRedirect bool) *Request { r.disableRedirect = disableRedirect return r } func (r *Request) DoRawRequest() bool { return r.doRawRequest } func (r *Request) SetDoRawRequest(doRawRequest bool) *Request { r.doRawRequest = doRawRequest return r } func (r *Request) DoRawClient() bool { return r.doRawClient } func (r *Request) SetDoRawClient(doRawClient bool) *Request { r.doRawClient = doRawClient return r } func (r *RequestOpts) SkipTLSVerify() bool { return r.skipTLSVerify } func (r *Request) SetSkipTLSVerify(skipTLSVerify bool) *Request { r.skipTLSVerify = skipTLSVerify return r } func (r *Request) TlsConfig() *tls.Config { return r.tlsConfig } func (r *Request) SetTlsConfig(tlsConfig *tls.Config) *Request { r.tlsConfig = tlsConfig return r } func (r *Request) AutoFetchRespBody() bool { return r.autoFetchRespBody } func (r *Request) SetAutoFetchRespBody(autoFetchRespBody bool) *Request { r.autoFetchRespBody = autoFetchRespBody return r } func (r *Request) ResetReqHeader() *Request { r.headers = make(http.Header) return r } func (r *Request) ResetReqCookies() *Request { r.cookies = []*http.Cookie{} return r } func (r *Request) AddSimpleCookie(key, value string) *Request { r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: "/"}) return r } func (r *Request) AddCookie(key, value, path string) *Request { r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: path}) return r } func (r *Request) AddFile(formName, filepath string) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), FileData: f, FileSize: stat.Size(), FileType: "application/octet-stream", }) return nil } func (r *Request) AddFileWithName(formName, filepath, filename string) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, FileName: filename, FileData: f, FileSize: stat.Size(), FileType: "application/octet-stream", }) return nil } func (r *Request) AddFileWithType(formName, filepath, filetype string) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), FileData: f, FileSize: stat.Size(), FileType: filetype, }) return nil } func (r *Request) AddFileWithNameAndType(formName, filepath, filename, filetype string) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, FileName: filename, FileData: f, FileSize: stat.Size(), FileType: filetype, }) return nil } func (r *Request) AddFileNoError(formName, filepath string) *Request { r.AddFile(formName, filepath) return r } func (r *Request) AddFileWithNameNoError(formName, filepath, filename string) *Request { r.AddFileWithName(formName, filepath, filename) return r } func (r *Request) AddFileWithTypeNoError(formName, filepath, filetype string) *Request { r.AddFileWithType(formName, filepath, filetype) return r } func (r *Request) AddFileWithNameAndTypeNoError(formName, filepath, filename, filetype string) *Request { r.AddFileWithNameAndType(formName, filepath, filename, filetype) return r } type RequestFile struct { FormName string FileName string FileData io.Reader FileSize int64 FileType string } type RequestOpt func(opt *RequestOpts) error // if doRawTransport is true, this function will nolonger work func WithDialTimeout(timeout time.Duration) RequestOpt { return func(opt *RequestOpts) error { opt.dialTimeout = timeout return nil } } // if doRawTransport is true, this function will nolonger work func WithTimeout(timeout time.Duration) RequestOpt { return func(opt *RequestOpts) error { opt.timeout = timeout return nil } } // if doRawTransport is true, this function will nolonger work func WithTlsConfig(tlscfg *tls.Config) RequestOpt { return func(opt *RequestOpts) error { opt.tlsConfig = tlscfg return nil } } // if doRawRequest is true, this function will nolonger work func WithHeader(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set(key, val) return nil } } // if doRawRequest is true, this function will nolonger work func WithHeaderMap(header map[string]string) RequestOpt { return func(opt *RequestOpts) error { for key, val := range header { opt.headers.Set(key, val) } return nil } } // if doRawRequest is true, this function will nolonger work func WithReader(r io.Reader) RequestOpt { return func(opt *RequestOpts) error { opt.bodyDataReader = r return nil } } // if doRawRequest is true, this function will nolonger work func WithBytes(r []byte) RequestOpt { return func(opt *RequestOpts) error { opt.bodyDataBytes = r return nil } } // if doRawRequest is true, this function will nolonger work func WithFormData(data map[string][]string) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFormData = data return nil } } // if doRawRequest is true, this function will nolonger work func WithFileDatas(data []RequestFile) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFileData = data return nil } } // if doRawRequest is true, this function will nolonger work func WithFileData(data RequestFile) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFileData = append(opt.bodyFileData, data) return nil } } // if doRawRequest is true, this function will nolonger work func WithAddFile(formName, filepath string) RequestOpt { return func(opt *RequestOpts) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), FileData: f, FileSize: stat.Size(), FileType: "application/octet-stream", }) return nil } } func WithAddFileWithName(formName, filepath, filename string) RequestOpt { return func(opt *RequestOpts) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: filename, FileData: f, FileSize: stat.Size(), FileType: "application/octet-stream", }) return nil } } func WithAddFileWithType(formName, filepath, filetype string) RequestOpt { return func(opt *RequestOpts) error { f, err := os.Open(filepath) if err != nil { return nil } stat, err := f.Stat() if err != nil { return nil } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), FileData: f, FileSize: stat.Size(), FileType: filetype, }) return nil } } func WithAddFileWithNameAndType(formName, filepath, filename, filetype string) RequestOpt { return func(opt *RequestOpts) error { f, err := os.Open(filepath) if err != nil { return err } stat, err := f.Stat() if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: filename, FileData: f, FileSize: stat.Size(), FileType: filetype, }) return nil } } func WithFetchRespBody(fetch bool) RequestOpt { return func(opt *RequestOpts) error { opt.autoFetchRespBody = fetch return nil } } func WithCookies(ck []*http.Cookie) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = ck return nil } } func WithCookie(key, val, path string) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path}) return nil } } func WithSimpleCookie(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: "/"}) return nil } } func WithCookieMap(header map[string]string, path string) RequestOpt { return func(opt *RequestOpts) error { for key, val := range header { opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path}) } return nil } } func WithQueries(queries map[string][]string) RequestOpt { return func(opt *RequestOpts) error { opt.queries = queries return nil } } func WithAddQueries(queries map[string]string) RequestOpt { return func(opt *RequestOpts) error { for k, v := range queries { opt.queries[k] = append(opt.queries[k], v) } return nil } } func WithAddQuery(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.queries[key] = append(opt.queries[key], val) return nil } } func WithProxy(proxy string) RequestOpt { return func(opt *RequestOpts) error { opt.proxy = proxy return nil } } func WithProcess(fn func(string, int64, int64)) RequestOpt { return func(opt *RequestOpts) error { opt.FileUploadRecallFn = fn return nil } } func WithContentType(ct string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set("Content-Type", ct) return nil } } func WithUserAgent(ua string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set("User-Agent", ua) return nil } } func WithSkipTLSVerify(skip bool) RequestOpt { return func(opt *RequestOpts) error { opt.skipTLSVerify = skip return nil } } func WithDisableRedirect(disable bool) RequestOpt { return func(opt *RequestOpts) error { opt.disableRedirect = disable return nil } } func WithDoRawRequest(doRawRequest bool) RequestOpt { return func(opt *RequestOpts) error { opt.doRawRequest = doRawRequest return nil } } func WithDoRawClient(doRawClient bool) RequestOpt { return func(opt *RequestOpts) error { opt.doRawClient = doRawClient return nil } } func WithDoRawTransport(doRawTrans bool) RequestOpt { return func(opt *RequestOpts) error { opt.doRawTransport = doRawTrans return nil } } func WithTransport(hs *http.Transport) RequestOpt { return func(opt *RequestOpts) error { opt.transport = hs return nil } } func WithRawRequest(req *http.Request) RequestOpt { return func(opt *RequestOpts) error { opt.rawRequest = req return nil } } func WithRawClient(hc *http.Client) RequestOpt { return func(opt *RequestOpts) error { opt.rawClient = hc return nil } } func WithCustomHostIP(ip []string) RequestOpt { return func(opt *RequestOpts) error { if len(ip) == 0 { return nil } for _, v := range ip { if net.ParseIP(v) == nil { return fmt.Errorf("invalid custom ip: %s", v) } } opt.customIP = ip return nil } } func WithAddCustomHostIP(ip string) RequestOpt { return func(opt *RequestOpts) error { if net.ParseIP(ip) == nil { return fmt.Errorf("invalid custom ip: %s", ip) } opt.customIP = append(opt.customIP, ip) return nil } } func WithLookUpFn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) RequestOpt { return func(opt *RequestOpts) error { if lookUpIPfn == nil { opt.alreadySetLookUpIPfn = false opt.lookUpIPfn = net.DefaultResolver.LookupIPAddr return nil } opt.lookUpIPfn = lookUpIPfn opt.alreadySetLookUpIPfn = true return nil } } // WithCustomDNS will use custom dns to resolve the host // Note: if LookUpIPfn is set, this function will not be used func WithCustomDNS(customDNS []string) RequestOpt { return func(opt *RequestOpts) error { for _, v := range customDNS { if net.ParseIP(v) == nil { return fmt.Errorf("invalid custom dns: %s", v) } } opt.customDNS = customDNS return nil } } // WithAddCustomDNS will use a custom dns to resolve the host // Note: if LookUpIPfn is set, this function will not be used func WithAddCustomDNS(customDNS string) RequestOpt { return func(opt *RequestOpts) error { if net.ParseIP(customDNS) == nil { return fmt.Errorf("invalid custom dns: %s", customDNS) } opt.customDNS = append(opt.customDNS, customDNS) return nil } } // WithAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body. // WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding. // also the memory usage will be higher // Note that this function will not work if doRawRequest is true func (r *RequestOpts) WithAutoCalcContentLength(autoCalcContentLength bool) RequestOpt { return func(opt *RequestOpts) error { r.autoCalcContentLength = autoCalcContentLength return nil } } type Response struct { *http.Response req Request data *Body } type Body struct { full []byte raw io.ReadCloser isFull bool } func (b *Body) readAll() { if !b.isFull { b.full, _ = io.ReadAll(b.raw) b.isFull = true b.raw.Close() } } func (b *Body) String() string { b.readAll() return string(b.full) } func (b *Body) Bytes() []byte { b.readAll() return b.full } func (b *Body) Unmarshal(u interface{}) error { b.readAll() return json.Unmarshal(b.full, u) } // Reader returns a reader for the body // if this function is called, other functions like String, Bytes, Unmarshal may not work func (b *Body) Reader() io.ReadCloser { if b.isFull { return io.NopCloser(bytes.NewReader(b.full)) } b.isFull = true return b.raw } func (b *Body) Close() error { return b.raw.Close() } func (r *Response) GetRequest() Request { return r.req } func (r *Response) Body() *Body { return r.data } func Curl(r *Request) (*Response, error) { r.errInfo = nil err := applyOptions(r) if err != nil { return nil, fmt.Errorf("apply options error: %s", err) } resp, err := r.rawClient.Do(r.rawRequest) var res = Response{ Response: resp, req: *r, data: new(Body), } if err != nil { res.Response = &http.Response{} return &res, fmt.Errorf("do request error: %s", err) } res.data.raw = resp.Body if r.autoFetchRespBody { res.data.full, _ = io.ReadAll(resp.Body) res.data.isFull = true resp.Body.Close() } return &res, r.errInfo } func NewReq(uri string, opts ...RequestOpt) *Request { return NewSimpleRequest(uri, "GET", opts...) } func NewReqWithContext(ctx context.Context, uri string, opts ...RequestOpt) *Request { return NewSimpleRequestWithContext(ctx, uri, "GET", opts...) } func NewSimpleRequest(uri string, method string, opts ...RequestOpt) *Request { r, _ := newRequest(context.Background(), uri, method, opts...) return r } func NewRequest(uri string, method string, opts ...RequestOpt) (*Request, error) { return newRequest(context.Background(), uri, method, opts...) } func NewSimpleRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) *Request { r, _ := newRequest(ctx, uri, method, opts...) return r } func NewRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) { return newRequest(ctx, uri, method, opts...) } func newRequest(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) { var req *http.Request var err error if method == "" { method = "GET" } method = strings.ToUpper(method) req, err = http.NewRequestWithContext(ctx, method, uri, nil) if err != nil { return nil, err } var r = &Request{ ctx: ctx, uri: uri, method: method, RequestOpts: RequestOpts{ rawRequest: req, rawClient: new(http.Client), timeout: DefaultTimeout, dialTimeout: DefaultDialTimeout, autoFetchRespBody: DefaultFetchRespBody, lookUpIPfn: net.DefaultResolver.LookupIPAddr, bodyFormData: make(map[string][]string), queries: make(map[string][]string), }, } r.headers = make(http.Header) if strings.ToUpper(method) == "POST" { r.headers.Set("Content-Type", HEADER_FORM_URLENCODE) } r.headers.Set("User-Agent", "B612 / 1.2.0") for _, v := range opts { if v != nil { err = v(&r.RequestOpts) if err != nil { return nil, err } } } if r.transport == nil { r.transport = &http.Transport{} } if r.doRawTransport { if r.skipTLSVerify { if r.transport.TLSClientConfig == nil { r.transport.TLSClientConfig = &tls.Config{} } r.transport.TLSClientConfig.InsecureSkipVerify = true } if r.tlsConfig != nil { r.transport.TLSClientConfig = r.tlsConfig } r.transport.DialContext = func(ctx context.Context, netType, addr string) (net.Conn, error) { var lastErr error var addrs []string host, port, err := net.SplitHostPort(addr) if err != nil { return nil, err } if len(r.customIP) > 0 { for _, v := range r.customIP { ipAddr := net.ParseIP(v) if ipAddr == nil { return nil, fmt.Errorf("invalid custom ip: %s", r.customIP) } tmpAddr := net.JoinHostPort(v, port) addrs = append(addrs, tmpAddr) } } else { ipLists, err := r.lookUpIPfn(ctx, host) if err != nil { return nil, err } for _, v := range ipLists { tmpAddr := net.JoinHostPort(v.String(), port) addrs = append(addrs, tmpAddr) } } for _, addr := range addrs { c, err := net.DialTimeout(netType, addr, r.dialTimeout) if err != nil { lastErr = err continue } if r.timeout != 0 { err = c.SetDeadline(time.Now().Add(r.timeout)) } return c, nil } return nil, lastErr } } return r, nil } func applyDataReader(r *Request) error { // 优先度为:bodyDataReader > bodyDataBytes > bodyFormData > bodyFileData if r.bodyDataReader != nil { r.rawRequest.Body = io.NopCloser(r.bodyDataReader) return nil } if len(r.bodyDataBytes) != 0 { r.rawRequest.Body = io.NopCloser(bytes.NewReader(r.bodyDataBytes)) return nil } if len(r.bodyFormData) != 0 && len(r.bodyFileData) == 0 { var body = url.Values{} for k, v := range r.bodyFormData { for _, vv := range v { body.Add(k, vv) } } r.rawRequest.Body = io.NopCloser(strings.NewReader(body.Encode())) return nil } if len(r.bodyFileData) != 0 { var pr, pw = io.Pipe() var w = multipart.NewWriter(pw) r.rawRequest.Header.Set("Content-Type", w.FormDataContentType()) go func() { defer pw.Close() // ensure pipe writer is closed if len(r.bodyFormData) != 0 { for k, v := range r.bodyFormData { for _, vv := range v { if err := w.WriteField(k, vv); err != nil { r.errInfo = err pw.CloseWithError(err) // close pipe with error return } } } } for _, v := range r.bodyFileData { var fw, err = w.CreateFormFile(v.FormName, v.FileName) if err != nil { r.errInfo = err pw.CloseWithError(err) // close pipe with error return } if _, err := copyWithContext(r.ctx, r.FileUploadRecallFn, v.FileName, v.FileSize, fw, v.FileData); err != nil { r.errInfo = err pw.CloseWithError(err) // close pipe with error return } } if err := w.Close(); err != nil { pw.CloseWithError(err) // close pipe with error if writer close fails } }() r.rawRequest.Body = pr return nil } return nil } func applyOptions(r *Request) error { defer func() { r.alreadyApply = true }() var req = r.rawRequest if !r.doRawRequest { if r.queries != nil { sid := req.URL.Query() for k, v := range r.queries { for _, vv := range v { sid.Add(k, vv) } } req.URL.RawQuery = sid.Encode() } for k, v := range r.headers { for _, vv := range v { req.Header.Add(k, vv) } } if len(r.cookies) != 0 { for _, v := range r.cookies { req.AddCookie(v) } } if r.basicAuth[0] != "" || r.basicAuth[1] != "" { req.SetBasicAuth(r.basicAuth[0], r.basicAuth[1]) } err := applyDataReader(r) if err != nil { return fmt.Errorf("apply data reader error: %s", err) } if r.autoCalcContentLength { if req.Body != nil { data, err := io.ReadAll(req.Body) if err != nil { return fmt.Errorf("read data error: %s", err) } req.Header.Set("Content-Length", strconv.Itoa(len(data))) req.Body = io.NopCloser(bytes.NewReader(data)) } } } if r.proxy != "" { purl, err := url.Parse(r.proxy) if err != nil { return fmt.Errorf("parse proxy url error: %s", err) } r.transport.Proxy = http.ProxyURL(purl) } if !r.doRawClient { if !r.doRawTransport { if !r.alreadySetLookUpIPfn && len(r.customIP) > 0 { resolver := net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { for _, addr := range r.customIP { if conn, err = net.Dial("udp", addr+":53"); err != nil { continue } else { return conn, nil } } return }, } r.lookUpIPfn = resolver.LookupIPAddr } } r.rawClient.Transport = r.transport if r.disableRedirect { r.rawClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } } } return nil } func copyWithContext(ctx context.Context, recall func(string, int64, int64), filename string, total int64, dst io.Writer, src io.Reader) (written int64, err error) { pr, pw := io.Pipe() defer pr.Close() go func() { defer pw.Close() _, err := io.Copy(pw, src) if err != nil { pw.CloseWithError(err) } }() var count int64 buf := make([]byte, 4096) for { select { case <-ctx.Done(): return written, ctx.Err() default: nr, err := pr.Read(buf) if err != nil { if err == io.EOF { go recall(filename, count, total) return written, nil } return written, err } count += int64(nr) if recall != nil { go recall(filename, count, total) } nw, err := dst.Write(buf[:nr]) if err != nil { return written, err } if nr != nw { return written, io.ErrShortWrite } written += int64(nr) } } }