starnet/curl.go

package starnet

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded`
	HEADER_FORM_DATA      = `multipart/form-data`
	HEADER_JSON           = `application/json`
	HEADER_PLAIN          = `text/plain`
)

var (
	DefaultDialTimeout   = 5 * time.Second
	DefaultTimeout       = 10 * time.Second
	DefaultFetchRespBody = false
)

func UrlEncodeRaw(str string) string {
	strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1)
	return strs
}

func UrlEncode(str string) string {
	return url.QueryEscape(str)
}

func UrlDecode(str string) (string, error) {
	return url.QueryUnescape(str)
}

func BuildQuery(queryData map[string]string) string {
	query := url.Values{}
	for k, v := range queryData {
		query.Add(k, v)
	}
	return query.Encode()
}

// BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string,
// and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests,
// where the server expects the request body to be URL-encoded form data.
//
// Parameters:
// queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request.
//
// Returns:
// A byte slice representing the URL-encoded form data.
func BuildPostForm(queryMap map[string]string) []byte {
	return []byte(BuildQuery(queryMap))
}

func Get(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "GET", opts...).Do()
}

func Post(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "POST", opts...).Do()
}

func Options(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "OPTIONS", opts...).Do()
}

func Put(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "PUT", opts...).Do()
}

func Delete(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "DELETE", opts...).Do()
}

func Head(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "HEAD", opts...).Do()
}

func Patch(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "PATCH", opts...).Do()
}

func Trace(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "TRACE", opts...).Do()
}

func Connect(uri string, opts ...RequestOpt) (*Response, error) {
	return NewSimpleRequest(uri, "CONNECT", opts...).Do()
}

type Request struct {
	ctx     context.Context
	uri     string
	method  string
	errInfo error
	RequestOpts
}

func (r *Request) Clone() *Request {
	clonedRequest := &Request{
		ctx:     r.ctx,
		uri:     r.uri,
		method:  r.method,
		errInfo: r.errInfo,
		RequestOpts: RequestOpts{
			headers:               CloneHeader(r.headers),
			cookies:               CloneCookies(r.cookies),
			bodyFormData:          CloneStringMapSlice(r.bodyFormData),
			bodyFileData:          CloneFiles(r.bodyFileData),
			queries:               CloneStringMapSlice(r.queries),
			bodyDataBytes:         CloneByteSlice(r.bodyDataBytes),
			proxy:                 r.proxy,
			timeout:               r.timeout,
			dialTimeout:           r.dialTimeout,
			alreadyApply:          r.alreadyApply,
			disableRedirect:       r.disableRedirect,
			doRawRequest:          r.doRawRequest,
			doRawClient:           r.doRawClient,
			doRawTransport:        r.doRawTransport,
			skipTLSVerify:         r.skipTLSVerify,
			autoFetchRespBody:     r.autoFetchRespBody,
			customIP:              CloneStringSlice(r.customIP),
			alreadySetLookUpIPfn:  r.alreadySetLookUpIPfn,
			lookUpIPfn:            r.lookUpIPfn,
			customDNS:             CloneStringSlice(r.customDNS),
			basicAuth:             r.basicAuth,
			autoCalcContentLength: r.autoCalcContentLength,
		},
	}

	// 手动深拷贝嵌套引用类型
	if r.bodyDataReader != nil {
		clonedRequest.bodyDataReader = r.bodyDataReader
	}

	if r.FileUploadRecallFn != nil {
		clonedRequest.FileUploadRecallFn = r.FileUploadRecallFn
	}

	// 对于 tlsConfig 类型，需要手动复制
	if r.tlsConfig != nil {
		clonedRequest.tlsConfig = CloneTLSConfig(r.tlsConfig)
	}

	// 对于 http.Transport，需要进行手动复制
	if r.transport != nil {
		clonedRequest.transport = CloneTransport(r.transport)
	}

	return clonedRequest
}

// CloneHeader 复制 http.Header
func CloneHeader(original http.Header) http.Header {
	newHeader := make(http.Header)
	for key, values := range original {
		copiedValues := make([]string, len(values))
		copy(copiedValues, values)
		newHeader[key] = copiedValues
	}
	return newHeader
}

// CloneCookies 复制 []*http.Cookie
func CloneCookies(original []*http.Cookie) []*http.Cookie {
	cloned := make([]*http.Cookie, len(original))
	for i, cookie := range original {
		cloned[i] = &http.Cookie{
			Name:       cookie.Name,
			Value:      cookie.Value,
			Path:       cookie.Path,
			Domain:     cookie.Domain,
			Expires:    cookie.Expires,
			RawExpires: cookie.RawExpires,
			MaxAge:     cookie.MaxAge,
			Secure:     cookie.Secure,
			HttpOnly:   cookie.HttpOnly,
			SameSite:   cookie.SameSite,
			Raw:        cookie.Raw,
			Unparsed:   append([]string(nil), cookie.Unparsed...),
		}
	}
	return cloned
}

// CloneStringMapSlice 复制 map[string][]string
func CloneStringMapSlice(original map[string][]string) map[string][]string {
	newMap := make(map[string][]string)
	for key, values := range original {
		copiedValues := make([]string, len(values))
		copy(copiedValues, values)
		newMap[key] = copiedValues
	}
	return newMap
}

// CloneFiles 复制 []RequestFile
func CloneFiles(original []RequestFile) []RequestFile {
	newFiles := make([]RequestFile, len(original))
	copy(newFiles, original)
	return newFiles
}

// CloneByteSlice 复制 []byte
func CloneByteSlice(original []byte) []byte {
	if original == nil {
		return nil
	}
	newSlice := make([]byte, len(original))
	copy(newSlice, original)
	return newSlice
}

// CloneStringSlice 复制 []string
func CloneStringSlice(original []string) []string {
	newSlice := make([]string, len(original))
	copy(newSlice, original)
	return newSlice
}

// CloneTLSConfig 复制 tls.Config
func CloneTLSConfig(original *tls.Config) *tls.Config {
	newConfig := &tls.Config{
		Rand:                        original.Rand,
		Time:                        original.Time,
		Certificates:                append([]tls.Certificate(nil), original.Certificates...),
		NameToCertificate:           original.NameToCertificate,
		GetCertificate:              original.GetCertificate,
		GetClientCertificate:        original.GetClientCertificate,
		GetConfigForClient:          original.GetConfigForClient,
		VerifyPeerCertificate:       original.VerifyPeerCertificate,
		VerifyConnection:            original.VerifyConnection,
		RootCAs:                     original.RootCAs,
		NextProtos:                  append([]string(nil), original.NextProtos...),
		ServerName:                  original.ServerName,
		ClientAuth:                  original.ClientAuth,
		ClientCAs:                   original.ClientCAs,
		InsecureSkipVerify:          original.InsecureSkipVerify,
		CipherSuites:                append([]uint16(nil), original.CipherSuites...),
		PreferServerCipherSuites:    original.PreferServerCipherSuites,
		SessionTicketsDisabled:      original.SessionTicketsDisabled,
		SessionTicketKey:            original.SessionTicketKey,
		ClientSessionCache:          original.ClientSessionCache,
		MinVersion:                  original.MinVersion,
		MaxVersion:                  original.MaxVersion,
		CurvePreferences:            append([]tls.CurveID(nil), original.CurvePreferences...),
		DynamicRecordSizingDisabled: original.DynamicRecordSizingDisabled,
		Renegotiation:               original.Renegotiation,
		KeyLogWriter:                original.KeyLogWriter,
	}
	return newConfig
}

// CloneTransport 复制 http.Transport
func CloneTransport(original *http.Transport) *http.Transport {
	newTransport := &http.Transport{
		Proxy:                  original.Proxy,
		DialContext:            original.DialContext,
		Dial:                   original.Dial,
		DialTLS:                original.DialTLS,
		TLSClientConfig:        original.TLSClientConfig,
		TLSHandshakeTimeout:    original.TLSHandshakeTimeout,
		DisableKeepAlives:      original.DisableKeepAlives,
		DisableCompression:     original.DisableCompression,
		MaxIdleConns:           original.MaxIdleConns,
		MaxIdleConnsPerHost:    original.MaxIdleConnsPerHost,
		IdleConnTimeout:        original.IdleConnTimeout,
		ResponseHeaderTimeout:  original.ResponseHeaderTimeout,
		ExpectContinueTimeout:  original.ExpectContinueTimeout,
		TLSNextProto:           original.TLSNextProto,
		ProxyConnectHeader:     original.ProxyConnectHeader,
		MaxResponseHeaderBytes: original.MaxResponseHeaderBytes,
		WriteBufferSize:        original.WriteBufferSize,
		ReadBufferSize:         original.ReadBufferSize,
	}
	return newTransport
}
func (r *Request) Method() string {
	return r.method
}

func (r *Request) SetMethod(method string) error {
	method = strings.ToUpper(method)
	if !validMethod(method) {
		return fmt.Errorf("invalid method: %s", method)
	}
	r.method = method
	r.rawRequest.Method = method
	return nil
}

func (r *Request) SetMethodNoError(method string) *Request {
	r.SetMethod(method)
	return r
}

func (r *Request) Uri() string {
	return r.uri
}

func (r *Request) SetUri(uri string) error {
	u, err := url.Parse(uri)
	if err != nil {
		return fmt.Errorf("parse uri error: %s", err)
	}
	r.uri = uri
	u.Host = removeEmptyPort(u.Host)
	r.rawRequest.Host = u.Host
	r.rawRequest.URL = u
	return nil
}

func (r *Request) SetUriNoError(uri string) *Request {
	r.SetUri(uri)
	return r
}

func (r *Request) RawRequest() *http.Request {
	return r.rawRequest
}

func (r *Request) SetRawRequest(rawRequest *http.Request) *Request {
	r.rawRequest = rawRequest
	return r
}

func (r *Request) RawClient() *http.Client {
	return r.rawClient
}

func (r *Request) SetRawClient(rawClient *http.Client) *Request {
	r.rawClient = rawClient
	return r
}

func (r *Request) Do() (*Response, error) {
	return Curl(r)
}

func (r *Request) Get() (*Response, error) {
	err := r.SetMethod("GET")
	if err != nil {
		return nil, err
	}
	return Curl(r)
}

func (r *Request) Post(data []byte) (*Response, error) {
	err := r.SetMethod("POST")
	if err != nil {
		return nil, err
	}
	r.bodyDataBytes = data
	r.bodyDataReader = nil
	return Curl(r)
}

type RequestOpts struct {
	rawRequest *http.Request
	rawClient  *http.Client

	alreadyApply   bool
	bodyDataBytes  []byte
	bodyDataReader io.Reader
	bodyFormData   map[string][]string
	bodyFileData   []RequestFile
	//以上优先度为 bodyDataReader> bodyDataBytes > bodyFormData > bodyFileData
	FileUploadRecallFn func(filename string, upPos int64, total int64)
	proxy              string
	timeout            time.Duration
	dialTimeout        time.Duration
	headers            http.Header
	cookies            []*http.Cookie
	transport          *http.Transport
	queries            map[string][]string
	disableRedirect    bool
	//doRawRequest=true 不对request修改，直接发送
	doRawRequest bool
	//doRawClient=true 不对http client修改，直接发送
	doRawClient bool
	//doRawTransPort=true 不对http transport修改，直接发送
	doRawTransport        bool
	skipTLSVerify         bool
	tlsConfig             *tls.Config
	autoFetchRespBody     bool
	customIP              []string
	alreadySetLookUpIPfn  bool
	lookUpIPfn            func(ctx context.Context, host string) ([]net.IPAddr, error)
	customDNS             []string
	basicAuth             [2]string
	autoCalcContentLength bool
}

func (r *Request) AutoCalcContentLength() bool {
	return r.autoCalcContentLength
}

// SetAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body.
// WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding.
// also the memory usage will be higher
// Note that this function will not work if doRawRequest is true
func (r *Request) SetAutoCalcContentLength(autoCalcContentLength bool) *Request {
	r.autoCalcContentLength = autoCalcContentLength
	return r
}

// BasicAuth returns the username and password provided in the request's Authorization header.
func (r *RequestOpts) BasicAuth() (string, string) {
	return r.basicAuth[0], r.basicAuth[1]
}

// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
// Note: If doRawRequest is true, this function will nolonger work
func (r *RequestOpts) SetBasicAuth(username, password string) *RequestOpts {
	r.basicAuth = [2]string{username, password}
	return r
}

func (r *Request) DoRawTransport() bool {
	return r.doRawTransport
}

func (r *Request) SetDoRawTransport(doRawTransport bool) *Request {
	r.doRawTransport = doRawTransport
	return r
}

func (r *Request) CustomDNS() []string {
	return r.customDNS
}

// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) SetCustomDNS(customDNS []string) error {
	for _, v := range customDNS {
		if net.ParseIP(v) == nil {
			return fmt.Errorf("invalid custom dns: %s", v)
		}
	}
	r.customDNS = customDNS
	return nil
}

// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) SetCustomDNSNoError(customDNS []string) *Request {
	r.SetCustomDNS(customDNS)
	return r
}

// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) AddCustomDNS(customDNS []string) error {
	for _, v := range customDNS {
		if net.ParseIP(v) == nil {
			return fmt.Errorf("invalid custom dns: %s", v)
		}
	}
	r.customDNS = customDNS
	return nil
}

// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) AddCustomDNSNoError(customDNS []string) *Request {
	r.AddCustomDNS(customDNS)
	return r
}

func (r *Request) LookUpIPfn() func(ctx context.Context, host string) ([]net.IPAddr, error) {
	return r.lookUpIPfn
}

func (r *Request) SetLookUpIPfn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) *Request {
	if lookUpIPfn == nil {
		r.alreadySetLookUpIPfn = false
		r.lookUpIPfn = net.DefaultResolver.LookupIPAddr
		return r
	}
	r.lookUpIPfn = lookUpIPfn
	r.alreadySetLookUpIPfn = true
	return r
}

func (r *Request) CustomHostIP() []string {
	return r.customIP
}

func (r *Request) SetCustomHostIP(customIP []string) *Request {
	r.customIP = customIP
	return r
}

func (r *Request) AddCustomHostIP(customIP string) *Request {
	r.customIP = append(r.customIP, customIP)
	return r
}

func (r *Request) BodyDataBytes() []byte {
	return r.bodyDataBytes
}

func (r *Request) SetBodyDataBytes(bodyDataBytes []byte) *Request {
	r.bodyDataBytes = bodyDataBytes
	return r
}

func (r *Request) BodyDataReader() io.Reader {
	return r.bodyDataReader
}

func (r *Request) SetBodyDataReader(bodyDataReader io.Reader) *Request {
	r.bodyDataReader = bodyDataReader
	return r
}

func (r *Request) BodyFormData() map[string][]string {
	return r.bodyFormData
}

func (r *Request) SetBodyFormData(bodyFormData map[string][]string) *Request {
	r.bodyFormData = bodyFormData
	return r
}

func (r *Request) SetFormData(bodyFormData map[string][]string) *Request {
	return r.SetBodyFormData(bodyFormData)
}

func (r *Request) AddFormMapData(bodyFormData map[string]string) *Request {
	for k, v := range bodyFormData {
		r.bodyFormData[k] = append(r.bodyFormData[k], v)
	}
	return r
}

func (r *Request) AddFormData(k, v string) *Request {
	r.bodyFormData[k] = append(r.bodyFormData[k], v)
	return r
}

func (r *Request) BodyFileData() []RequestFile {
	return r.bodyFileData
}

func (r *Request) SetBodyFileData(bodyFileData []RequestFile) *Request {
	r.bodyFileData = bodyFileData
	return r
}

func (r *Request) Proxy() string {
	return r.proxy
}

func (r *Request) SetProxy(proxy string) *Request {
	r.proxy = proxy
	return r
}

func (r *Request) Timeout() time.Duration {
	return r.timeout
}

func (r *Request) SetTimeout(timeout time.Duration) *Request {
	r.timeout = timeout
	return r
}

func (r *Request) DialTimeout() time.Duration {
	return r.dialTimeout
}

func (r *Request) SetDialTimeout(dialTimeout time.Duration) *Request {
	r.dialTimeout = dialTimeout
	return r
}

func (r *Request) Headers() http.Header {
	return r.headers
}

func (r *Request) SetHeaders(headers http.Header) *Request {
	r.headers = headers
	return r
}

func (r *Request) AddHeader(key, val string) *Request {
	r.headers.Add(key, val)
	return r
}

func (r *Request) SetHeader(key, val string) *Request {
	r.headers.Set(key, val)
	return r
}

func (r *Request) SetContentType(ct string) *Request {
	r.headers.Set("Content-Type", ct)
	return r
}

func (r *Request) SetUserAgent(ua string) *Request {
	r.headers.Set("User-Agent", ua)
	return r
}

func (r *Request) DeleteHeader(key string) *Request {
	r.headers.Del(key)
	return r
}

func (r *Request) Cookies() []*http.Cookie {
	return r.cookies
}

func (r *Request) SetCookies(cookies []*http.Cookie) *Request {
	r.cookies = cookies
	return r
}

func (r *Request) Transport() *http.Transport {
	return r.transport
}

func (r *Request) SetTransport(transport *http.Transport) *Request {
	r.transport = transport
	return r
}

func (r *Request) Queries() map[string][]string {
	return r.queries
}

func (r *Request) SetQueries(queries map[string][]string) *Request {
	r.queries = queries
	return r
}

func (r *Request) AddQueries(queries map[string]string) *Request {
	for k, v := range queries {
		r.queries[k] = append(r.queries[k], v)
	}
	return r
}

func (r *Request) AddQuery(key, value string) *Request {
	r.queries[key] = append(r.queries[key], value)
	return r
}

func (r *Request) DelQueryKv(key, value string) *Request {
	if _, ok := r.queries[key]; !ok {
		return r
	}
	for i, v := range r.queries[key] {
		if v == value {
			r.queries[key] = append(r.queries[key][:i], r.queries[key][i+1:]...)
		}
	}
	return r
}

func (r *Request) DelQuery(key string) *Request {
	if _, ok := r.queries[key]; !ok {
		return r
	}
	delete(r.queries, key)
	return r
}

func (r *Request) DisableRedirect() bool {
	return r.disableRedirect
}

func (r *Request) SetDisableRedirect(disableRedirect bool) *Request {
	r.disableRedirect = disableRedirect
	return r
}

func (r *Request) DoRawRequest() bool {
	return r.doRawRequest
}

func (r *Request) SetDoRawRequest(doRawRequest bool) *Request {
	r.doRawRequest = doRawRequest
	return r
}

func (r *Request) DoRawClient() bool {
	return r.doRawClient
}

func (r *Request) SetDoRawClient(doRawClient bool) *Request {
	r.doRawClient = doRawClient
	return r
}

func (r *RequestOpts) SkipTLSVerify() bool {
	return r.skipTLSVerify
}

func (r *Request) SetSkipTLSVerify(skipTLSVerify bool) *Request {
	r.skipTLSVerify = skipTLSVerify
	return r
}

func (r *Request) TlsConfig() *tls.Config {
	return r.tlsConfig
}

func (r *Request) SetTlsConfig(tlsConfig *tls.Config) *Request {
	r.tlsConfig = tlsConfig
	return r
}

func (r *Request) AutoFetchRespBody() bool {
	return r.autoFetchRespBody
}

func (r *Request) SetAutoFetchRespBody(autoFetchRespBody bool) *Request {
	r.autoFetchRespBody = autoFetchRespBody
	return r
}

func (r *Request) ResetReqHeader() *Request {
	r.headers = make(http.Header)
	return r
}

func (r *Request) ResetReqCookies() *Request {
	r.cookies = []*http.Cookie{}
	return r
}

func (r *Request) AddSimpleCookie(key, value string) *Request {
	r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: "/"})
	return r
}

func (r *Request) AddCookie(key, value, path string) *Request {
	r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: path})
	return r
}

func (r *Request) AddFile(formName, filepath string) error {
	f, err := os.Open(filepath)
	if err != nil {
		return err
	}
	stat, err := f.Stat()
	if err != nil {
		return err
	}
	r.bodyFileData = append(r.bodyFileData, RequestFile{
		FormName: formName,
		FileName: stat.Name(),
		FileData: f,
		FileSize: stat.Size(),
		FileType: "application/octet-stream",
	})
	return nil
}

func (r *Request) AddFileWithName(formName, filepath, filename string) error {
	f, err := os.Open(filepath)
	if err != nil {
		return err
	}
	stat, err := f.Stat()
	if err != nil {
		return err
	}
	r.bodyFileData = append(r.bodyFileData, RequestFile{
		FormName: formName,
		FileName: filename,
		FileData: f,
		FileSize: stat.Size(),
		FileType: "application/octet-stream",
	})
	return nil
}

func (r *Request) AddFileWithType(formName, filepath, filetype string) error {
	f, err := os.Open(filepath)
	if err != nil {
		return err
	}
	stat, err := f.Stat()
	if err != nil {
		return err
	}
	r.bodyFileData = append(r.bodyFileData, RequestFile{
		FormName: formName,
		FileName: stat.Name(),
		FileData: f,
		FileSize: stat.Size(),
		FileType: filetype,
	})
	return nil
}
func (r *Request) AddFileWithNameAndType(formName, filepath, filename, filetype string) error {
	f, err := os.Open(filepath)
	if err != nil {
		return err
	}
	stat, err := f.Stat()
	if err != nil {
		return err
	}
	r.bodyFileData = append(r.bodyFileData, RequestFile{
		FormName: formName,
		FileName: filename,
		FileData: f,
		FileSize: stat.Size(),
		FileType: filetype,
	})
	return nil
}

func (r *Request) AddFileNoError(formName, filepath string) *Request {
	r.AddFile(formName, filepath)
	return r
}

func (r *Request) AddFileWithNameNoError(formName, filepath, filename string) *Request {
	r.AddFileWithName(formName, filepath, filename)
	return r
}

func (r *Request) AddFileWithTypeNoError(formName, filepath, filetype string) *Request {
	r.AddFileWithType(formName, filepath, filetype)
	return r
}
func (r *Request) AddFileWithNameAndTypeNoError(formName, filepath, filename, filetype string) *Request {
	r.AddFileWithNameAndType(formName, filepath, filename, filetype)
	return r
}

type RequestFile struct {
	FormName string
	FileName string
	FileData io.Reader
	FileSize int64
	FileType string
}

type RequestOpt func(opt *RequestOpts) error

// if doRawTransport is true, this function will nolonger work
func WithDialTimeout(timeout time.Duration) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.dialTimeout = timeout
		return nil
	}
}

// if doRawTransport is true, this function will nolonger work
func WithTimeout(timeout time.Duration) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.timeout = timeout
		return nil
	}
}

// if doRawTransport is true, this function will nolonger work
func WithTlsConfig(tlscfg *tls.Config) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.tlsConfig = tlscfg
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithHeader(key, val string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.headers.Set(key, val)
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithHeaderMap(header map[string]string) RequestOpt {
	return func(opt *RequestOpts) error {
		for key, val := range header {
			opt.headers.Set(key, val)
		}
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithReader(r io.Reader) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.bodyDataReader = r
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithBytes(r []byte) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.bodyDataBytes = r
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithFormData(data map[string][]string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.bodyFormData = data
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithFileDatas(data []RequestFile) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.bodyFileData = data
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithFileData(data RequestFile) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.bodyFileData = append(opt.bodyFileData, data)
		return nil
	}
}

// if doRawRequest is true, this function will nolonger work
func WithAddFile(formName, filepath string) RequestOpt {
	return func(opt *RequestOpts) error {
		f, err := os.Open(filepath)
		if err != nil {
			return err
		}
		stat, err := f.Stat()
		if err != nil {
			return err
		}
		opt.bodyFileData = append(opt.bodyFileData, RequestFile{
			FormName: formName,
			FileName: stat.Name(),
			FileData: f,
			FileSize: stat.Size(),
			FileType: "application/octet-stream",
		})
		return nil
	}
}

func WithAddFileWithName(formName, filepath, filename string) RequestOpt {
	return func(opt *RequestOpts) error {
		f, err := os.Open(filepath)
		if err != nil {
			return err
		}
		stat, err := f.Stat()
		if err != nil {
			return err
		}
		opt.bodyFileData = append(opt.bodyFileData, RequestFile{
			FormName: formName,
			FileName: filename,
			FileData: f,
			FileSize: stat.Size(),
			FileType: "application/octet-stream",
		})
		return nil
	}
}

func WithAddFileWithType(formName, filepath, filetype string) RequestOpt {
	return func(opt *RequestOpts) error {
		f, err := os.Open(filepath)
		if err != nil {
			return nil
		}
		stat, err := f.Stat()
		if err != nil {
			return nil
		}
		opt.bodyFileData = append(opt.bodyFileData, RequestFile{
			FormName: formName,
			FileName: stat.Name(),
			FileData: f,
			FileSize: stat.Size(),
			FileType: filetype,
		})
		return nil
	}
}

func WithAddFileWithNameAndType(formName, filepath, filename, filetype string) RequestOpt {
	return func(opt *RequestOpts) error {
		f, err := os.Open(filepath)
		if err != nil {
			return err
		}
		stat, err := f.Stat()
		if err != nil {
			return err
		}
		opt.bodyFileData = append(opt.bodyFileData, RequestFile{
			FormName: formName,
			FileName: filename,
			FileData: f,
			FileSize: stat.Size(),
			FileType: filetype,
		})
		return nil
	}
}

func WithFetchRespBody(fetch bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.autoFetchRespBody = fetch
		return nil
	}
}

func WithCookies(ck []*http.Cookie) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.cookies = ck
		return nil
	}
}

func WithCookie(key, val, path string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path})
		return nil
	}
}

func WithSimpleCookie(key, val string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: "/"})
		return nil
	}
}

func WithCookieMap(header map[string]string, path string) RequestOpt {
	return func(opt *RequestOpts) error {
		for key, val := range header {
			opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path})
		}
		return nil
	}
}

func WithQueries(queries map[string][]string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.queries = queries
		return nil
	}
}

func WithAddQueries(queries map[string]string) RequestOpt {
	return func(opt *RequestOpts) error {
		for k, v := range queries {
			opt.queries[k] = append(opt.queries[k], v)
		}
		return nil
	}
}

func WithAddQuery(key, val string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.queries[key] = append(opt.queries[key], val)
		return nil
	}
}

func WithProxy(proxy string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.proxy = proxy
		return nil
	}
}

func WithProcess(fn func(string, int64, int64)) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.FileUploadRecallFn = fn
		return nil
	}
}

func WithContentType(ct string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.headers.Set("Content-Type", ct)
		return nil
	}
}

func WithUserAgent(ua string) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.headers.Set("User-Agent", ua)
		return nil
	}
}

func WithSkipTLSVerify(skip bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.skipTLSVerify = skip
		return nil
	}
}

func WithDisableRedirect(disable bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.disableRedirect = disable
		return nil
	}
}

func WithDoRawRequest(doRawRequest bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.doRawRequest = doRawRequest
		return nil
	}
}

func WithDoRawClient(doRawClient bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.doRawClient = doRawClient
		return nil
	}
}

func WithDoRawTransport(doRawTrans bool) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.doRawTransport = doRawTrans
		return nil
	}
}

func WithTransport(hs *http.Transport) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.transport = hs
		return nil
	}
}

func WithRawRequest(req *http.Request) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.rawRequest = req
		return nil
	}
}

func WithRawClient(hc *http.Client) RequestOpt {
	return func(opt *RequestOpts) error {
		opt.rawClient = hc
		return nil
	}
}

func WithCustomHostIP(ip []string) RequestOpt {
	return func(opt *RequestOpts) error {
		if len(ip) == 0 {
			return nil
		}
		for _, v := range ip {
			if net.ParseIP(v) == nil {
				return fmt.Errorf("invalid custom ip: %s", v)
			}
		}
		opt.customIP = ip
		return nil
	}
}

func WithAddCustomHostIP(ip string) RequestOpt {
	return func(opt *RequestOpts) error {
		if net.ParseIP(ip) == nil {
			return fmt.Errorf("invalid custom ip: %s", ip)
		}
		opt.customIP = append(opt.customIP, ip)
		return nil
	}
}

func WithLookUpFn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) RequestOpt {
	return func(opt *RequestOpts) error {
		if lookUpIPfn == nil {
			opt.alreadySetLookUpIPfn = false
			opt.lookUpIPfn = net.DefaultResolver.LookupIPAddr
			return nil
		}
		opt.lookUpIPfn = lookUpIPfn
		opt.alreadySetLookUpIPfn = true
		return nil
	}
}

// WithCustomDNS will use custom dns to resolve the host
// Note: if LookUpIPfn is set, this function will not be used
func WithCustomDNS(customDNS []string) RequestOpt {
	return func(opt *RequestOpts) error {
		for _, v := range customDNS {
			if net.ParseIP(v) == nil {
				return fmt.Errorf("invalid custom dns: %s", v)
			}
		}
		opt.customDNS = customDNS
		return nil
	}
}

// WithAddCustomDNS will use a custom dns to resolve the host
// Note: if LookUpIPfn is set, this function will not be used
func WithAddCustomDNS(customDNS string) RequestOpt {
	return func(opt *RequestOpts) error {
		if net.ParseIP(customDNS) == nil {
			return fmt.Errorf("invalid custom dns: %s", customDNS)
		}
		opt.customDNS = append(opt.customDNS, customDNS)
		return nil
	}
}

// WithAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body.
// WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding.
// also the memory usage will be higher
// Note that this function will not work if doRawRequest is true
func (r *RequestOpts) WithAutoCalcContentLength(autoCalcContentLength bool) RequestOpt {
	return func(opt *RequestOpts) error {
		r.autoCalcContentLength = autoCalcContentLength
		return nil
	}
}

type Response struct {
	*http.Response
	req  Request
	data *Body
}

type Body struct {
	full   []byte
	raw    io.ReadCloser
	isFull bool
	sync.Mutex
}

func (b *Body) readAll() {
	b.Lock()
	defer b.Unlock()
	if !b.isFull {
		if b.raw == nil {
			b.isFull = true
			return
		}
		b.full, _ = io.ReadAll(b.raw)
		b.isFull = true
		b.raw.Close()
	}
}

func (b *Body) String() string {
	b.readAll()
	return string(b.full)
}

func (b *Body) Bytes() []byte {
	b.readAll()
	return b.full
}

func (b *Body) Unmarshal(u interface{}) error {
	b.readAll()
	return json.Unmarshal(b.full, u)
}

// Reader returns a reader for the body
// if this function is called, other functions like String, Bytes, Unmarshal may not work
func (b *Body) Reader() io.ReadCloser {
	b.Lock()
	defer b.Unlock()
	if b.isFull {
		return io.NopCloser(bytes.NewReader(b.full))
	}
	b.isFull = true
	return b.raw
}

func (b *Body) Close() error {
	return b.raw.Close()
}

func (r *Response) GetRequest() Request {
	return r.req
}

func (r *Response) Body() *Body {
	return r.data
}

func Curl(r *Request) (*Response, error) {
	r.errInfo = nil
	err := applyOptions(r)
	if err != nil {
		return nil, fmt.Errorf("apply options error: %s", err)
	}
	resp, err := r.rawClient.Do(r.rawRequest)
	var res = Response{
		Response: resp,
		req:      *r,
		data:     new(Body),
	}
	if err != nil {
		res.Response = &http.Response{}
		return &res, fmt.Errorf("do request error: %s", err)
	}
	res.data.raw = resp.Body
	if r.autoFetchRespBody {
		res.data.full, _ = io.ReadAll(resp.Body)
		res.data.isFull = true
		resp.Body.Close()
	}
	return &res, r.errInfo
}

func NewReq(uri string, opts ...RequestOpt) *Request {
	return NewSimpleRequest(uri, "GET", opts...)
}

func NewReqWithContext(ctx context.Context, uri string, opts ...RequestOpt) *Request {
	return NewSimpleRequestWithContext(ctx, uri, "GET", opts...)
}

func NewSimpleRequest(uri string, method string, opts ...RequestOpt) *Request {
	r, _ := newRequest(context.Background(), uri, method, opts...)
	return r
}

func NewRequest(uri string, method string, opts ...RequestOpt) (*Request, error) {
	return newRequest(context.Background(), uri, method, opts...)
}

func NewSimpleRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) *Request {
	r, _ := newRequest(ctx, uri, method, opts...)
	return r
}

func NewRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) {
	return newRequest(ctx, uri, method, opts...)
}

func newRequest(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) {
	var req *http.Request
	var err error
	if method == "" {
		method = "GET"
	}
	method = strings.ToUpper(method)
	req, err = http.NewRequestWithContext(ctx, method, uri, nil)
	if err != nil {
		return nil, err
	}
	var r = &Request{
		ctx:    ctx,
		uri:    uri,
		method: method,
		RequestOpts: RequestOpts{
			rawRequest:        req,
			rawClient:         new(http.Client),
			timeout:           DefaultTimeout,
			dialTimeout:       DefaultDialTimeout,
			autoFetchRespBody: DefaultFetchRespBody,
			lookUpIPfn:        net.DefaultResolver.LookupIPAddr,
			bodyFormData:      make(map[string][]string),
			queries:           make(map[string][]string),
		},
	}

	r.headers = make(http.Header)
	if strings.ToUpper(method) == "POST" {
		r.headers.Set("Content-Type", HEADER_FORM_URLENCODE)
	}
	r.headers.Set("User-Agent", "B612 / 1.2.0")
	for _, v := range opts {
		if v != nil {
			err = v(&r.RequestOpts)
			if err != nil {
				return nil, err
			}
		}
	}
	if r.transport == nil {
		r.transport = &http.Transport{}
	}
	if r.doRawTransport {
		if r.skipTLSVerify {
			if r.transport.TLSClientConfig == nil {
				r.transport.TLSClientConfig = &tls.Config{}
			}
			r.transport.TLSClientConfig.InsecureSkipVerify = true
		}
		if r.tlsConfig != nil {
			r.transport.TLSClientConfig = r.tlsConfig
		}
		r.transport.DialContext = func(ctx context.Context, netType, addr string) (net.Conn, error) {
			var lastErr error
			var addrs []string
			host, port, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}
			if len(r.customIP) > 0 {
				for _, v := range r.customIP {
					ipAddr := net.ParseIP(v)
					if ipAddr == nil {
						return nil, fmt.Errorf("invalid custom ip: %s", r.customIP)
					}
					tmpAddr := net.JoinHostPort(v, port)
					addrs = append(addrs, tmpAddr)
				}
			} else {
				ipLists, err := r.lookUpIPfn(ctx, host)
				if err != nil {
					return nil, err
				}
				for _, v := range ipLists {
					tmpAddr := net.JoinHostPort(v.String(), port)
					addrs = append(addrs, tmpAddr)
				}
			}
			for _, addr := range addrs {
				c, err := net.DialTimeout(netType, addr, r.dialTimeout)
				if err != nil {
					lastErr = err
					continue
				}
				if r.timeout != 0 {
					err = c.SetDeadline(time.Now().Add(r.timeout))
				}
				return c, nil
			}
			return nil, lastErr
		}
	}
	return r, nil
}

func applyDataReader(r *Request) error {
	// 优先度为：bodyDataReader > bodyDataBytes > bodyFormData > bodyFileData
	if r.bodyDataReader != nil {
		r.rawRequest.Body = io.NopCloser(r.bodyDataReader)
		return nil
	}
	if len(r.bodyDataBytes) != 0 {
		r.rawRequest.Body = io.NopCloser(bytes.NewReader(r.bodyDataBytes))
		return nil
	}
	if len(r.bodyFormData) != 0 && len(r.bodyFileData) == 0 {
		var body = url.Values{}
		for k, v := range r.bodyFormData {
			for _, vv := range v {
				body.Add(k, vv)
			}
		}
		r.rawRequest.Body = io.NopCloser(strings.NewReader(body.Encode()))
		return nil
	}
	if len(r.bodyFileData) != 0 {
		var pr, pw = io.Pipe()
		var w = multipart.NewWriter(pw)
		r.rawRequest.Header.Set("Content-Type", w.FormDataContentType())
		go func() {
			defer pw.Close() // ensure pipe writer is closed

			if len(r.bodyFormData) != 0 {
				for k, v := range r.bodyFormData {
					for _, vv := range v {
						if err := w.WriteField(k, vv); err != nil {
							r.errInfo = err
							pw.CloseWithError(err) // close pipe with error
							return
						}
					}
				}
			}
			for _, v := range r.bodyFileData {
				var fw, err = w.CreateFormFile(v.FormName, v.FileName)
				if err != nil {
					r.errInfo = err
					pw.CloseWithError(err) // close pipe with error
					return
				}
				if _, err := copyWithContext(r.ctx, r.FileUploadRecallFn, v.FileName, v.FileSize, fw, v.FileData); err != nil {
					r.errInfo = err
					pw.CloseWithError(err) // close pipe with error
					return
				}
			}

			if err := w.Close(); err != nil {
				pw.CloseWithError(err) // close pipe with error if writer close fails
			}
		}()

		r.rawRequest.Body = pr
		return nil
	}
	return nil
}

func applyOptions(r *Request) error {
	defer func() {
		r.alreadyApply = true
	}()
	var req = r.rawRequest
	if !r.doRawRequest {
		if r.queries != nil {
			sid := req.URL.Query()
			for k, v := range r.queries {
				for _, vv := range v {
					sid.Add(k, vv)
				}
			}
			req.URL.RawQuery = sid.Encode()
		}
		for k, v := range r.headers {
			for _, vv := range v {
				req.Header.Add(k, vv)
			}
		}
		if len(r.cookies) != 0 {
			for _, v := range r.cookies {
				req.AddCookie(v)
			}
		}
		if r.basicAuth[0] != "" || r.basicAuth[1] != "" {
			req.SetBasicAuth(r.basicAuth[0], r.basicAuth[1])
		}
		err := applyDataReader(r)
		if err != nil {
			return fmt.Errorf("apply data reader error: %s", err)
		}
		if r.autoCalcContentLength {
			if req.Body != nil {
				data, err := io.ReadAll(req.Body)
				if err != nil {
					return fmt.Errorf("read data error: %s", err)
				}
				req.Header.Set("Content-Length", strconv.Itoa(len(data)))
				req.Body = io.NopCloser(bytes.NewReader(data))
			}
		}
	}
	if r.proxy != "" {
		purl, err := url.Parse(r.proxy)
		if err != nil {
			return fmt.Errorf("parse proxy url error: %s", err)
		}
		r.transport.Proxy = http.ProxyURL(purl)
	}
	if !r.doRawClient {
		if !r.doRawTransport {
			if !r.alreadySetLookUpIPfn && len(r.customIP) > 0 {
				resolver := net.Resolver{
					PreferGo: true,
					Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) {
						for _, addr := range r.customIP {
							if conn, err = net.Dial("udp", addr+":53"); err != nil {
								continue
							} else {
								return conn, nil
							}
						}
						return
					},
				}
				r.lookUpIPfn = resolver.LookupIPAddr
			}
		}
		r.rawClient.Transport = r.transport
		if r.disableRedirect {
			r.rawClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			}
		}
	}
	return nil
}

func copyWithContext(ctx context.Context, recall func(string, int64, int64), filename string, total int64, dst io.Writer, src io.Reader) (written int64, err error) {
	pr, pw := io.Pipe()
	defer pr.Close()

	go func() {
		defer pw.Close()
		_, err := io.Copy(pw, src)
		if err != nil {
			pw.CloseWithError(err)
		}
	}()
	var count int64
	buf := make([]byte, 4096)
	for {
		select {
		case <-ctx.Done():
			return written, ctx.Err()
		default:
			nr, err := pr.Read(buf)
			if err != nil {
				if err == io.EOF {
					go recall(filename, count, total)
					return written, nil
				}
				return written, err
			}
			count += int64(nr)
			if recall != nil {
				go recall(filename, count, total)
			}
			nw, err := dst.Write(buf[:nr])
			if err != nil {
				return written, err
			}
			if nr != nw {
				return written, io.ErrShortWrite
			}
			written += int64(nr)
		}
	}
}