From 260ceb90ed7691282aac715db0d9a77d4c632f49 Mon Sep 17 00:00:00 2001 From: starainrt Date: Thu, 21 Aug 2025 15:02:02 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84http=20Client=E9=83=A8?= =?UTF-8?q?=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- curl.go | 797 +++++++++++++++++++++++----------------------- curl_default.go | 197 ++++++++++++ curl_test.go | 18 +- curl_transport.go | 134 ++++++++ 4 files changed, 743 insertions(+), 403 deletions(-) create mode 100644 curl_default.go create mode 100644 curl_transport.go diff --git a/curl.go b/curl.go index 70761d7..5270d23 100644 --- a/curl.go +++ b/curl.go @@ -18,91 +18,9 @@ import ( "time" ) -const ( - HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded` - HEADER_FORM_DATA = `multipart/form-data` - HEADER_JSON = `application/json` - HEADER_PLAIN = `text/plain` -) - -var ( - DefaultDialTimeout = 5 * time.Second - DefaultTimeout = 10 * time.Second - DefaultFetchRespBody = false -) - -func UrlEncodeRaw(str string) string { - strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1) - return strs -} - -func UrlEncode(str string) string { - return url.QueryEscape(str) -} - -func UrlDecode(str string) (string, error) { - return url.QueryUnescape(str) -} - -func BuildQuery(queryData map[string]string) string { - query := url.Values{} - for k, v := range queryData { - query.Add(k, v) - } - return query.Encode() -} - -// BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string, -// and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests, -// where the server expects the request body to be URL-encoded form data. -// -// Parameters: -// queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request. -// -// Returns: -// A byte slice representing the URL-encoded form data. -func BuildPostForm(queryMap map[string]string) []byte { - return []byte(BuildQuery(queryMap)) -} - -func Get(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "GET", opts...).Do() -} - -func Post(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "POST", opts...).Do() -} - -func Options(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "OPTIONS", opts...).Do() -} - -func Put(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "PUT", opts...).Do() -} - -func Delete(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "DELETE", opts...).Do() -} - -func Head(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "HEAD", opts...).Do() -} - -func Patch(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "PATCH", opts...).Do() -} - -func Trace(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "TRACE", opts...).Do() -} - -func Connect(uri string, opts ...RequestOpt) (*Response, error) { - return NewSimpleRequest(uri, "CONNECT", opts...).Do() -} - type Request struct { ctx context.Context + doCtx context.Context // 用于在请求中传递上下文信息 uri string method string errInfo error @@ -127,10 +45,7 @@ func (r *Request) Clone() *Request { dialTimeout: r.dialTimeout, dialFn: r.dialFn, alreadyApply: r.alreadyApply, - disableRedirect: r.disableRedirect, doRawRequest: r.doRawRequest, - doRawClient: r.doRawClient, - doRawTransport: r.doRawTransport, skipTLSVerify: r.skipTLSVerify, autoFetchRespBody: r.autoFetchRespBody, customIP: CloneStringSlice(r.customIP), @@ -141,31 +56,26 @@ func (r *Request) Clone() *Request { autoCalcContentLength: r.autoCalcContentLength, }, } - if r.doRawClient { - clonedRequest.rawClient = r.rawClient - } else { - clonedRequest.rawClient = new(http.Client) - } - if r.doRawRequest { - clonedRequest.rawRequest = r.rawRequest - } + clonedRequest.rawClient = r.rawClient + // 手动深拷贝嵌套引用类型 if r.bodyDataReader != nil { clonedRequest.bodyDataReader = r.bodyDataReader } - if r.FileUploadRecallFn != nil { - clonedRequest.FileUploadRecallFn = r.FileUploadRecallFn + if r.fileUploadRecallFn != nil { + clonedRequest.fileUploadRecallFn = r.fileUploadRecallFn } // 对于 tlsConfig 类型，需要手动复制 if r.tlsConfig != nil { - clonedRequest.tlsConfig = CloneTLSConfig(r.tlsConfig) + clonedRequest.tlsConfig = r.tlsConfig.Clone() } - - // 对于 http.Transport，需要进行手动复制 if r.transport != nil { - clonedRequest.transport = CloneTransport(r.transport) + clonedRequest.transport = r.transport + } + if r.doRawRequest { + clonedRequest.rawRequest = r.rawRequest } if clonedRequest.rawRequest == nil { @@ -242,63 +152,6 @@ func CloneStringSlice(original []string) []string { return newSlice } -// CloneTLSConfig 复制 tls.Config -func CloneTLSConfig(original *tls.Config) *tls.Config { - newConfig := &tls.Config{ - Rand: original.Rand, - Time: original.Time, - Certificates: append([]tls.Certificate(nil), original.Certificates...), - NameToCertificate: original.NameToCertificate, - GetCertificate: original.GetCertificate, - GetClientCertificate: original.GetClientCertificate, - GetConfigForClient: original.GetConfigForClient, - VerifyPeerCertificate: original.VerifyPeerCertificate, - VerifyConnection: original.VerifyConnection, - RootCAs: original.RootCAs, - NextProtos: append([]string(nil), original.NextProtos...), - ServerName: original.ServerName, - ClientAuth: original.ClientAuth, - ClientCAs: original.ClientCAs, - InsecureSkipVerify: original.InsecureSkipVerify, - CipherSuites: append([]uint16(nil), original.CipherSuites...), - PreferServerCipherSuites: original.PreferServerCipherSuites, - SessionTicketsDisabled: original.SessionTicketsDisabled, - SessionTicketKey: original.SessionTicketKey, - ClientSessionCache: original.ClientSessionCache, - MinVersion: original.MinVersion, - MaxVersion: original.MaxVersion, - CurvePreferences: append([]tls.CurveID(nil), original.CurvePreferences...), - DynamicRecordSizingDisabled: original.DynamicRecordSizingDisabled, - Renegotiation: original.Renegotiation, - KeyLogWriter: original.KeyLogWriter, - } - return newConfig -} - -// CloneTransport 复制 http.Transport -func CloneTransport(original *http.Transport) *http.Transport { - newTransport := &http.Transport{ - Proxy: original.Proxy, - DialContext: original.DialContext, - Dial: original.Dial, - DialTLS: original.DialTLS, - TLSClientConfig: original.TLSClientConfig, - TLSHandshakeTimeout: original.TLSHandshakeTimeout, - DisableKeepAlives: original.DisableKeepAlives, - DisableCompression: original.DisableCompression, - MaxIdleConns: original.MaxIdleConns, - MaxIdleConnsPerHost: original.MaxIdleConnsPerHost, - IdleConnTimeout: original.IdleConnTimeout, - ResponseHeaderTimeout: original.ResponseHeaderTimeout, - ExpectContinueTimeout: original.ExpectContinueTimeout, - TLSNextProto: original.TLSNextProto, - ProxyConnectHeader: original.ProxyConnectHeader, - MaxResponseHeaderBytes: original.MaxResponseHeaderBytes, - WriteBufferSize: original.WriteBufferSize, - ReadBufferSize: original.ReadBufferSize, - } - return newTransport -} func (r *Request) Method() string { return r.method } @@ -323,6 +176,9 @@ func (r *Request) Uri() string { } func (r *Request) SetUri(uri string) error { + if r.doRawRequest { + return fmt.Errorf("doRawRequest is true, cannot set uri") + } u, err := url.Parse(uri) if err != nil { return fmt.Errorf("parse uri error: %s", err) @@ -331,6 +187,9 @@ func (r *Request) SetUri(uri string) error { u.Host = removeEmptyPort(u.Host) r.rawRequest.Host = u.Host r.rawRequest.URL = u + if r.tlsConfig != nil { + r.tlsConfig.ServerName = u.Hostname() + } return nil } @@ -357,10 +216,12 @@ func (r *Request) SetRawClient(rawClient *http.Client) *Request { return r } +// Do sends the HTTP request and returns the response. func (r *Request) Do() (*Response, error) { return Curl(r) } +// Get sends a GET request to the specified URI and returns the response. func (r *Request) Get() (*Response, error) { err := r.SetMethod("GET") if err != nil { @@ -369,6 +230,7 @@ func (r *Request) Get() (*Response, error) { return Curl(r) } +// Post sends a POST request with the provided data to the specified URI and returns the response. func (r *Request) Post(data []byte) (*Response, error) { err := r.SetMethod("POST") if err != nil { @@ -380,8 +242,10 @@ func (r *Request) Post(data []byte) (*Response, error) { } type RequestOpts struct { - rawRequest *http.Request - rawClient *http.Client + rawRequest *http.Request + rawClient *http.Client + transport *http.Transport + customTransport bool alreadyApply bool bodyDataBytes []byte @@ -389,22 +253,17 @@ type RequestOpts struct { bodyFormData map[string][]string bodyFileData []RequestFile //以上优先度为 bodyDataReader> bodyDataBytes > bodyFormData > bodyFileData - FileUploadRecallFn func(filename string, upPos int64, total int64) + fileUploadRecallFn func(filename string, upPos int64, total int64) proxy string timeout time.Duration dialTimeout time.Duration dialFn func(ctx context.Context, network, addr string) (net.Conn, error) headers http.Header cookies []*http.Cookie - transport *http.Transport - queries map[string][]string - disableRedirect bool + + queries map[string][]string //doRawRequest=true 不对request修改，直接发送 - doRawRequest bool - //doRawClient=true 不对http client修改，直接发送 - doRawClient bool - //doRawTransPort=true 不对http transport修改，直接发送 - doRawTransport bool + doRawRequest bool skipTLSVerify bool tlsConfig *tls.Config autoFetchRespBody bool @@ -416,10 +275,27 @@ type RequestOpts struct { autoCalcContentLength bool } +func (r *RequestOpts) CustomTransport() bool { + return r.customTransport +} + +func (r *RequestOpts) SetCustomTransport(customTransport bool) { + r.customTransport = customTransport +} + +func (r *RequestOpts) FileUploadRecallFn() func(filename string, upPos int64, total int64) { + return r.fileUploadRecallFn +} + +func (r *RequestOpts) SetFileUploadRecallFn(FileUploadRecallFn func(filename string, upPos int64, total int64)) { + r.fileUploadRecallFn = FileUploadRecallFn +} + func (r *Request) DialFn() func(ctx context.Context, network, addr string) (net.Conn, error) { return r.dialFn } +// SetDialFn sets the dial function for the request. func (r *Request) SetDialFn(dialFn func(ctx context.Context, network, addr string) (net.Conn, error)) { r.dialFn = dialFn } @@ -432,8 +308,16 @@ func (r *Request) AutoCalcContentLength() bool { // WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding. // also the memory usage will be higher // Note that this function will not work if doRawRequest is true -func (r *Request) SetAutoCalcContentLength(autoCalcContentLength bool) *Request { +func (r *Request) SetAutoCalcContentLength(autoCalcContentLength bool) error { + if r.doRawRequest { + return fmt.Errorf("doRawRequest is true, cannot set autoCalcContentLength") + } r.autoCalcContentLength = autoCalcContentLength + return nil +} + +func (r *Request) SetAutoCalcContentLengthNoError(autoCalcContentLength bool) *Request { + r.SetAutoCalcContentLength(autoCalcContentLength) return r } @@ -449,20 +333,13 @@ func (r *RequestOpts) SetBasicAuth(username, password string) *RequestOpts { return r } -func (r *Request) DoRawTransport() bool { - return r.doRawTransport -} - -func (r *Request) SetDoRawTransport(doRawTransport bool) *Request { - r.doRawTransport = doRawTransport - return r -} - func (r *Request) CustomDNS() []string { return r.customDNS } -// Note: if LookUpIPfn is set, this function will not be used +// SetCustomDNS sets the custom DNS servers for the request. +// Note: if LookUpIPfn is set, this function will not be used. +// if use custom Transport Dialer, this function will not work by default,but if the *http.Client is create by this package, it will work func (r *Request) SetCustomDNS(customDNS []string) error { for _, v := range customDNS { if net.ParseIP(v) == nil { @@ -473,13 +350,17 @@ func (r *Request) SetCustomDNS(customDNS []string) error { return nil } -// Note: if LookUpIPfn is set, this function will not be used +// SetCustomDNSNoError sets the custom DNS servers for the request. +// Note: if LookUpIPfn is set, this function will not be used. +// if use custom Transport Dialer, this function will not work by default,but if the *http.Client is create by this package, it will work func (r *Request) SetCustomDNSNoError(customDNS []string) *Request { r.SetCustomDNS(customDNS) return r } -// Note: if LookUpIPfn is set, this function will not be used +// AddCustomDNS adds custom DNS servers to the request. +// Note: if LookUpIPfn is set, this function will not be used. +// if use custom Transport Dialer, this function will not work by default,but if the *http.Client is create by this package, it will work func (r *Request) AddCustomDNS(customDNS []string) error { for _, v := range customDNS { if net.ParseIP(v) == nil { @@ -490,7 +371,9 @@ func (r *Request) AddCustomDNS(customDNS []string) error { return nil } -// Note: if LookUpIPfn is set, this function will not be used +// AddCustomDNSNoError adds custom DNS servers to the request. +// Note: if LookUpIPfn is set, this function will not be used. +// if use custom Transport Dialer, this function will not work by default,but if the *http.Client is create by this package, it will work func (r *Request) AddCustomDNSNoError(customDNS []string) *Request { r.AddCustomDNS(customDNS) return r @@ -500,6 +383,10 @@ func (r *Request) LookUpIPfn() func(ctx context.Context, host string) ([]net.IPA return r.lookUpIPfn } +// SetLookUpIPfn sets the function used to look up IP addresses for a given host. +// If lookUpIPfn is nil, it will use the default resolver's LookupIPAddr function. +// Note: if use custom Transport Dialer, this function will not work by default,but if the *http.Client is create by this package, it will work +// Note: if CustomHostIP is set, this function will not be used. func (r *Request) SetLookUpIPfn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) *Request { if lookUpIPfn == nil { r.alreadySetLookUpIPfn = false @@ -511,51 +398,79 @@ func (r *Request) SetLookUpIPfn(lookUpIPfn func(ctx context.Context, host string return r } +// CustomHostIP returns the custom IP addresses used for the request. func (r *Request) CustomHostIP() []string { return r.customIP } +// SetCustomHostIP sets the custom IP addresses used for the request. +// if you want to use a specific IP address for a host without DNS resolution, you can set this. +// Set nil to clear the custom IP addresses. +// Note: lookUpIPfn will not be used if customIP is set. func (r *Request) SetCustomHostIP(customIP []string) *Request { r.customIP = customIP return r } +// AddCustomHostIP adds a custom IP address to the request. func (r *Request) AddCustomHostIP(customIP string) *Request { r.customIP = append(r.customIP, customIP) return r } +// BodyDataBytes returns the raw body data as a byte slice. func (r *Request) BodyDataBytes() []byte { return r.bodyDataBytes } +// SetBodyDataBytes sets the raw body data for the request. +// The priority order of the data is: bodyDataReader > **bodyDataBytes** > bodyFormData > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetBodyDataBytes(bodyDataBytes []byte) *Request { r.bodyDataBytes = bodyDataBytes return r } +// BodyDataReader returns the raw body data as an io.Reader. func (r *Request) BodyDataReader() io.Reader { return r.bodyDataReader } +// SetBodyDataReader sets the raw body data for the request as an io.Reader. +// The priority order of the data is: **bodyDataReader** > bodyDataBytes > bodyFormData > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetBodyDataReader(bodyDataReader io.Reader) *Request { r.bodyDataReader = bodyDataReader return r } +// BodyFormData returns the form data as a map of string slices. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) BodyFormData() map[string][]string { return r.bodyFormData } +// SetBodyFormData sets the form data for the request. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetBodyFormData(bodyFormData map[string][]string) *Request { r.bodyFormData = bodyFormData return r } +// SetFormData is an alias for SetBodyFormData. +// It allows you to set form data in the request body. +// This is useful when you want to use a more descriptive name for the function. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetFormData(bodyFormData map[string][]string) *Request { return r.SetBodyFormData(bodyFormData) } +// AddFormMapData adds form data from a map to the request body. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) AddFormMapData(bodyFormData map[string]string) *Request { for k, v := range bodyFormData { r.bodyFormData[k] = append(r.bodyFormData[k], v) @@ -563,108 +478,151 @@ func (r *Request) AddFormMapData(bodyFormData map[string]string) *Request { return r } +// AddFormData adds a single key-value pair to the form data in the request body. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// Note: If doRawRequest is true, this function will not work. func (r *Request) AddFormData(k, v string) *Request { r.bodyFormData[k] = append(r.bodyFormData[k], v) return r } +// BodyFileData returns the file data as a slice of RequestFile. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// Note: If doRawRequest is true, this function will not work. func (r *Request) BodyFileData() []RequestFile { return r.bodyFileData } +// SetBodyFileData sets the file data for the request. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetBodyFileData(bodyFileData []RequestFile) *Request { r.bodyFileData = bodyFileData return r } +// Proxy returns the proxy URL for the request. func (r *Request) Proxy() string { return r.proxy } +// SetProxy sets the proxy URL for the request. func (r *Request) SetProxy(proxy string) *Request { r.proxy = proxy return r } +// Timeout returns the timeout duration for the request. func (r *Request) Timeout() time.Duration { return r.timeout } +// SetTimeout sets the timeout duration for the request. func (r *Request) SetTimeout(timeout time.Duration) *Request { r.timeout = timeout return r } +// DialTimeout returns the dial timeout duration for the request. func (r *Request) DialTimeout() time.Duration { return r.dialTimeout } +// SetDialTimeout sets the dial timeout duration for the request. func (r *Request) SetDialTimeout(dialTimeout time.Duration) *Request { r.dialTimeout = dialTimeout return r } +// Headers returns the request headers as an http.Header. func (r *Request) Headers() http.Header { return r.headers } +// SetHeaders sets the request headers using an http.Header. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetHeaders(headers http.Header) *Request { r.headers = headers return r } +// AddHeader adds a single header to the request. +// This function will append the header if it already exists. +// Note: If doRawRequest is true, this function will not work. func (r *Request) AddHeader(key, val string) *Request { r.headers.Add(key, val) return r } +// SetHeader sets a single header in the request. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetHeader(key, val string) *Request { r.headers.Set(key, val) return r } -func (r *Request) SetContentType(ct string) *Request { - r.headers.Set("Content-Type", ct) - return r -} - -func (r *Request) SetUserAgent(ua string) *Request { - r.headers.Set("User-Agent", ua) - return r -} - +// DeleteHeader removes a header from the request. +// if the header has multiple values, it will remove all values for that header. +// Note: If doRawRequest is true, this function will not work. func (r *Request) DeleteHeader(key string) *Request { r.headers.Del(key) return r } +// SetContentType sets the Content-Type header for the request. +// This function will overwrite any existing Content-Type header. +// Note: If doRawRequest is true, this function will not work. +func (r *Request) SetContentType(ct string) *Request { + r.headers.Set("Content-Type", ct) + return r +} + +// SetUserAgent sets the User-Agent header for the request. +// This function will overwrite any existing User-Agent header. +// Note: If doRawRequest is true, this function will not work. +func (r *Request) SetUserAgent(ua string) *Request { + r.headers.Set("User-Agent", ua) + return r +} + +// Cookies returns the request cookies as a slice of http.Cookie. func (r *Request) Cookies() []*http.Cookie { return r.cookies } +// SetCookies sets the request cookies using a slice of http.Cookie. +// you can also use SetHeader("Cookie", "cookie1=value1; cookie2=value2") to set cookies. +// Note: If doRawRequest is true, this function will not work. func (r *Request) SetCookies(cookies []*http.Cookie) *Request { r.cookies = cookies return r } +// Transport returns the http.Transport used for the request. func (r *Request) Transport() *http.Transport { return r.transport } +// SetTransport set the http.Transport used for the request. +// Note: If doRawClient is true, this function will not work. func (r *Request) SetTransport(transport *http.Transport) *Request { r.transport = transport + r.customTransport = true return r } +// Queries returns the request queries as a map of string slices. func (r *Request) Queries() map[string][]string { return r.queries } +// SetQueries sets the request queries using a map of string slices. func (r *Request) SetQueries(queries map[string][]string) *Request { r.queries = queries return r } +// AddQueries adds multiple query parameters to the request. func (r *Request) AddQueries(queries map[string]string) *Request { for k, v := range queries { r.queries[k] = append(r.queries[k], v) @@ -672,11 +630,13 @@ func (r *Request) AddQueries(queries map[string]string) *Request { return r } +// AddQuery adds a single query parameter to the request. func (r *Request) AddQuery(key, value string) *Request { r.queries[key] = append(r.queries[key], value) return r } +// DelQueryKv removes a specific value from a query parameter. func (r *Request) DelQueryKv(key, value string) *Request { if _, ok := r.queries[key]; !ok { return r @@ -689,6 +649,7 @@ func (r *Request) DelQueryKv(key, value string) *Request { return r } +// DelQuery removes a query parameter from the request. func (r *Request) DelQuery(key string) *Request { if _, ok := r.queries[key]; !ok { return r @@ -697,81 +658,83 @@ func (r *Request) DelQuery(key string) *Request { return r } -func (r *Request) DisableRedirect() bool { - return r.disableRedirect -} - -func (r *Request) SetDisableRedirect(disableRedirect bool) *Request { - r.disableRedirect = disableRedirect - return r -} - +// DoRawRequest returns whether the request will be sent as a raw request. func (r *Request) DoRawRequest() bool { return r.doRawRequest } +// SetDoRawRequest sets whether the request will be sent as a raw request without any modifications. +// you can use this with function SetRawRequest to set a custom http.Request. func (r *Request) SetDoRawRequest(doRawRequest bool) *Request { r.doRawRequest = doRawRequest return r } -func (r *Request) DoRawClient() bool { - return r.doRawClient -} - -func (r *Request) SetDoRawClient(doRawClient bool) *Request { - r.doRawClient = doRawClient - return r -} - +// SkipTLSVerify returns whether the request will skip TLS verification. func (r *RequestOpts) SkipTLSVerify() bool { return r.skipTLSVerify } -// SetSkipTLSVerify This function will Not Work when use rawClient,use SetClientSkipVerify instead +// SetSkipTLSVerify Sets whether the request will skip TLS verification. func (r *Request) SetSkipTLSVerify(skipTLSVerify bool) *Request { r.skipTLSVerify = skipTLSVerify return r } +// TlsConfig returns the TLS configuration used for the request. func (r *Request) TlsConfig() *tls.Config { return r.tlsConfig } +// SetTlsConfig sets the TLS configuration for the request. +// Note: If you use SetSkipTLSVerify function, it will automatically set the InsecureSkipVerify field to true in the tls.Config. func (r *Request) SetTlsConfig(tlsConfig *tls.Config) *Request { r.tlsConfig = tlsConfig return r } +// AutoFetchRespBody returns whether the response body will be automatically fetched. func (r *Request) AutoFetchRespBody() bool { return r.autoFetchRespBody } +// SetAutoFetchRespBody sets whether the response body will be automatically fetched after the request is sent. +// If set to true, the response body will be read and stored in the Response object. +// if the body is too large, it may cause high memory usage. +// If set to false, you will need to manually read the response body using Response.Body() method. func (r *Request) SetAutoFetchRespBody(autoFetchRespBody bool) *Request { r.autoFetchRespBody = autoFetchRespBody return r } +// ResetReqHeader resets the request headers to an empty http.Header. func (r *Request) ResetReqHeader() *Request { r.headers = make(http.Header) return r } +// ResetReqCookies resets the request cookies to an empty slice. func (r *Request) ResetReqCookies() *Request { r.cookies = []*http.Cookie{} return r } +// AddSimpleCookie add a key-value cookie to the request. +// the path will be set to "/" func (r *Request) AddSimpleCookie(key, value string) *Request { r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: "/"}) return r } +// AddCookie adds a cookie to the request with the specified key, value, and path. func (r *Request) AddCookie(key, value, path string) *Request { r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: path}) return r } +// AddFile adds a file to the request with the specified form name and file path. +// The file will be read and uploaded as a multipart/form-data request. +// the file type will be set to "application/octet-stream" by default. func (r *Request) AddFile(formName, filepath string) error { stat, err := os.Stat(filepath) if err != nil { @@ -788,6 +751,9 @@ func (r *Request) AddFile(formName, filepath string) error { return nil } +// AddFileStream adds a file to the request with the specified form name, filename, size, and io.Reader stream. +// The file will be read and uploaded as a multipart/form-data request. +// the file type will be set to "application/octet-stream" by default. func (r *Request) AddFileStream(formName, filename string, size int64, stream io.Reader) error { r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, @@ -799,6 +765,10 @@ func (r *Request) AddFileStream(formName, filename string, size int64, stream io return nil } +// AddFileWithName adds a file to the request with the specified form name, file path, and filename. +// you can specify a custom filename for the file being uploaded. +// The file will be read and uploaded as a multipart/form-data request. +// the file type will be set to "application/octet-stream" by default. func (r *Request) AddFileWithName(formName, filepath, filename string) error { stat, err := os.Stat(filepath) if err != nil { @@ -815,6 +785,9 @@ func (r *Request) AddFileWithName(formName, filepath, filename string) error { return nil } +// AddFileWithType adds a file to the request with the specified form name, file path, and file type. +// you can specify a custom file type for the file being uploaded. +// The file will be read and uploaded as a multipart/form-data request. func (r *Request) AddFileWithType(formName, filepath, filetype string) error { stat, err := os.Stat(filepath) if err != nil { @@ -830,6 +803,10 @@ func (r *Request) AddFileWithType(formName, filepath, filetype string) error { }) return nil } + +// AddFileWithNameAndType adds a file to the request with the specified form name, file path, filename, and file type. +// you can specify a custom filename and file type for the file being uploaded. +// The file will be read and uploaded as a multipart/form-data request. func (r *Request) AddFileWithNameAndType(formName, filepath, filename, filetype string) error { stat, err := os.Stat(filepath) if err != nil { @@ -846,6 +823,9 @@ func (r *Request) AddFileWithNameAndType(formName, filepath, filename, filetype return nil } +// AddFileStreamWithType adds a file to the request with the specified form name, filename, file type, size, and io.Reader stream. +// The file will be read and uploaded as a multipart/form-data request. +// you can specify a custom file type for the file being uploaded. func (r *Request) AddFileStreamWithType(formName, filename, filetype string, size int64, stream io.Reader) error { r.bodyFileData = append(r.bodyFileData, RequestFile{ FormName: formName, @@ -857,35 +837,55 @@ func (r *Request) AddFileStreamWithType(formName, filename, filetype string, siz return nil } +// AddFileNoError adds a file to the request with the specified form name and file path. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileNoError(formName, filepath string) *Request { r.AddFile(formName, filepath) return r } +// AddFileWithNameNoError adds a file to the request with the specified form name, file path, and filename. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileWithNameNoError(formName, filepath, filename string) *Request { r.AddFileWithName(formName, filepath, filename) return r } +// AddFileWithTypeNoError adds a file to the request with the specified form name, file path, and file type. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileWithTypeNoError(formName, filepath, filetype string) *Request { r.AddFileWithType(formName, filepath, filetype) return r } + +// AddFileWithNameAndTypeNoError adds a file to the request with the specified form name, file path, filename, and file type. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileWithNameAndTypeNoError(formName, filepath, filename, filetype string) *Request { r.AddFileWithNameAndType(formName, filepath, filename, filetype) return r } +// AddFileStreamNoError adds a file to the request with the specified form name, filename, size, and io.Reader stream. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileStreamNoError(formName, filename string, size int64, stream io.Reader) *Request { r.AddFileStream(formName, filename, size, stream) return r } +// AddFileStreamWithTypeNoError adds a file to the request with the specified form name, filename, file type, size, and io.Reader stream. +// It will not return an error if the file cannot be added. +// this function is useful for chaining methods without error handling. func (r *Request) AddFileStreamWithTypeNoError(formName, filename, filetype string, size int64, stream io.Reader) *Request { r.AddFileStreamWithType(formName, filename, filetype, size, stream) return r } +// HttpClient returns the http.Client used for the request. func (r *Request) HttpClient() (*http.Client, error) { err := applyOptions(r) if err != nil { @@ -905,7 +905,8 @@ type RequestFile struct { type RequestOpt func(opt *RequestOpts) error -// if doRawTransport is true, this function will nolonger work +// WithDialTimeout sets the dial timeout for the request. +// If use custom Transport Dialer, this function will nolonger work. func WithDialTimeout(timeout time.Duration) RequestOpt { return func(opt *RequestOpts) error { opt.dialTimeout = timeout @@ -913,7 +914,9 @@ func WithDialTimeout(timeout time.Duration) RequestOpt { } } -// if doRawTransport is true, this function will nolonger work +// WithDial sets a custom dial function for the request. +// functions like WithDialTimeout will nolonger work if this function is used. +// If use custom Transport Dialer, this function will nolonger work. func WithDial(fn func(ctx context.Context, network string, addr string) (net.Conn, error)) RequestOpt { return func(opt *RequestOpts) error { opt.dialFn = fn @@ -921,7 +924,8 @@ func WithDial(fn func(ctx context.Context, network string, addr string) (net.Con } } -// if doRawTransport is true, this function will nolonger work +// WithTimeout sets the timeout for the request. +// If use custom Transport Dialer, this function will nolonger work. func WithTimeout(timeout time.Duration) RequestOpt { return func(opt *RequestOpts) error { opt.timeout = timeout @@ -929,7 +933,8 @@ func WithTimeout(timeout time.Duration) RequestOpt { } } -// if doRawTransport is true, this function will nolonger work +// WithTlsConfig sets the TLS configuration for the request. +// If use custom Transport Dialer, this function will nolonger work. func WithTlsConfig(tlscfg *tls.Config) RequestOpt { return func(opt *RequestOpts) error { opt.tlsConfig = tlscfg @@ -937,7 +942,8 @@ func WithTlsConfig(tlscfg *tls.Config) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithHeaders sets the request headers using an http.Header. +// If doRawRequest is true, this function will not work. func WithHeader(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set(key, val) @@ -945,7 +951,8 @@ func WithHeader(key, val string) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithHeaderMap sets the request headers using a map of string to string. +// If doRawRequest is true, this function will not work. func WithHeaderMap(header map[string]string) RequestOpt { return func(opt *RequestOpts) error { for key, val := range header { @@ -955,7 +962,9 @@ func WithHeaderMap(header map[string]string) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithReader sets the request body data using an io.Reader. +// The priority order of the data is: **bodyDataReader** > bodyDataBytes > bodyFormData > bodyFileData. +// If doRawRequest is true, this function will nolonger work. func WithReader(r io.Reader) RequestOpt { return func(opt *RequestOpts) error { opt.bodyDataReader = r @@ -963,7 +972,9 @@ func WithReader(r io.Reader) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithBytes sets the request body data using a byte slice. +// The priority order of the data is: bodyDataReader > **bodyDataBytes** > bodyFormData > bodyFileData. +// If doRawRequest is true, this function will nolonger work. func WithBytes(r []byte) RequestOpt { return func(opt *RequestOpts) error { opt.bodyDataBytes = r @@ -971,7 +982,9 @@ func WithBytes(r []byte) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithFormData sets the request body data using a map of string slices. +// The priority order of the data is: bodyDataReader > bodyDataBytes > **bodyFormData** > bodyFileData. +// If doRawRequest is true, this function will nolonger work. func WithFormData(data map[string][]string) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFormData = data @@ -979,7 +992,9 @@ func WithFormData(data map[string][]string) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithFileDatas sets the request body file data using a slice of RequestFile. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// If doRawRequest is true, this function will nolonger work. func WithFileDatas(data []RequestFile) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFileData = data @@ -987,7 +1002,9 @@ func WithFileDatas(data []RequestFile) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithFileData sets the request body file data using a single RequestFile. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// If doRawRequest is true, this function will nolonger work. func WithFileData(data RequestFile) RequestOpt { return func(opt *RequestOpts) error { opt.bodyFileData = append(opt.bodyFileData, data) @@ -995,42 +1012,44 @@ func WithFileData(data RequestFile) RequestOpt { } } -// if doRawRequest is true, this function will nolonger work +// WithAddFile adds a file to the request with the specified form name and file path. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// The file will be read and uploaded as a multipart/form-data request. +// the file type will be set to "application/octet-stream" by default. +// If doRawRequest is true, this function will nolonger work. func WithAddFile(formName, filepath string) RequestOpt { return func(opt *RequestOpts) error { - f, err := os.Open(filepath) - if err != nil { - return err - } - stat, err := f.Stat() + stat, err := os.Stat(filepath) if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), - FileData: f, + FileData: nil, FileSize: stat.Size(), FileType: "application/octet-stream", + FilePath: filepath, }) return nil } } +// WithAddFileWithName adds a file to the request with the specified form name, file path, and filename. +// you can specify a custom filename for the file being uploaded. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// The file will be read and uploaded as a multipart/form-data request. +// If doRawRequest is true, this function will nolonger work. func WithAddFileWithName(formName, filepath, filename string) RequestOpt { return func(opt *RequestOpts) error { - f, err := os.Open(filepath) - if err != nil { - return err - } - stat, err := f.Stat() + stat, err := os.Stat(filepath) if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: filename, - FileData: f, + FileData: nil, FileSize: stat.Size(), FileType: "application/octet-stream", }) @@ -1038,20 +1057,20 @@ func WithAddFileWithName(formName, filepath, filename string) RequestOpt { } } +// WithAddFileWithType adds a file to the request with the specified form name, file path, and file type. +// you can specify a custom file type for the file being uploaded. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// The file will be read and uploaded as a multipart/form-data request. func WithAddFileWithType(formName, filepath, filetype string) RequestOpt { return func(opt *RequestOpts) error { - f, err := os.Open(filepath) + stat, err := os.Stat(filepath) if err != nil { - return nil - } - stat, err := f.Stat() - if err != nil { - return nil + return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: stat.Name(), - FileData: f, + FileData: nil, FileSize: stat.Size(), FileType: filetype, }) @@ -1059,20 +1078,20 @@ func WithAddFileWithType(formName, filepath, filetype string) RequestOpt { } } +// WithAddFileWithNameAndType adds a file to the request with the specified form name, file path, filename, and file type. +// you can specify a custom filename and file type for the file being uploaded. +// The priority order of the data is: bodyDataReader > bodyDataBytes > bodyFormData > **bodyFileData**. +// The file will be read and uploaded as a multipart/form-data request. func WithAddFileWithNameAndType(formName, filepath, filename, filetype string) RequestOpt { return func(opt *RequestOpts) error { - f, err := os.Open(filepath) - if err != nil { - return err - } - stat, err := f.Stat() + stat, err := os.Stat(filepath) if err != nil { return err } opt.bodyFileData = append(opt.bodyFileData, RequestFile{ FormName: formName, FileName: filename, - FileData: f, + FileData: nil, FileSize: stat.Size(), FileType: filetype, }) @@ -1080,6 +1099,10 @@ func WithAddFileWithNameAndType(formName, filepath, filename, filetype string) R } } +// WithFetchRespBody sets whether the response body will be automatically fetched after the request is sent. +// If set to true, the response body will be read and stored in the Response object. +// If the body is too large, it may cause high memory usage. +// If set to false, you will need to manually read the response body using Response.Body() method. func WithFetchRespBody(fetch bool) RequestOpt { return func(opt *RequestOpts) error { opt.autoFetchRespBody = fetch @@ -1087,6 +1110,7 @@ func WithFetchRespBody(fetch bool) RequestOpt { } } +// WithCookies sets the request cookies using a slice of http.Cookie. func WithCookies(ck []*http.Cookie) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = ck @@ -1094,6 +1118,7 @@ func WithCookies(ck []*http.Cookie) RequestOpt { } } +// WithCookie adds a cookie to the request with the specified key, value, and path. func WithCookie(key, val, path string) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path}) @@ -1101,6 +1126,7 @@ func WithCookie(key, val, path string) RequestOpt { } } +// WithSimpleCookie adds a simple cookie to the request with the specified key and value. func WithSimpleCookie(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: "/"}) @@ -1108,6 +1134,7 @@ func WithSimpleCookie(key, val string) RequestOpt { } } +// WithCookieMap sets the request cookies using a map of string to string. func WithCookieMap(header map[string]string, path string) RequestOpt { return func(opt *RequestOpts) error { for key, val := range header { @@ -1117,6 +1144,7 @@ func WithCookieMap(header map[string]string, path string) RequestOpt { } } +// WithQueries sets the request queries using a map of string slices. func WithQueries(queries map[string][]string) RequestOpt { return func(opt *RequestOpts) error { opt.queries = queries @@ -1124,6 +1152,8 @@ func WithQueries(queries map[string][]string) RequestOpt { } } +// WithAddQueries adds multiple query parameters to the request using a map of string to string slices. +// if the key already exists, it will append the value to the existing slice. func WithAddQueries(queries map[string]string) RequestOpt { return func(opt *RequestOpts) error { for k, v := range queries { @@ -1133,6 +1163,7 @@ func WithAddQueries(queries map[string]string) RequestOpt { } } +// WithAddQuery adds a single query parameter to the request. func WithAddQuery(key, val string) RequestOpt { return func(opt *RequestOpts) error { opt.queries[key] = append(opt.queries[key], val) @@ -1140,6 +1171,7 @@ func WithAddQuery(key, val string) RequestOpt { } } +// WithProxy sets the proxy URL for the request. func WithProxy(proxy string) RequestOpt { return func(opt *RequestOpts) error { opt.proxy = proxy @@ -1147,13 +1179,22 @@ func WithProxy(proxy string) RequestOpt { } } +// WithProcess sets a callback function to process the file upload progress. +// The callback function will be called with the file name, uploaded bytes and total bytes. +// example: +// +// WithProcess(func(name string, uploaded int64, total int64) { +// fmt.Printf("Uploading %s: %d/%d bytes\n", name, uploaded, total) +// }) func WithProcess(fn func(string, int64, int64)) RequestOpt { return func(opt *RequestOpts) error { - opt.FileUploadRecallFn = fn + opt.fileUploadRecallFn = fn return nil } } +// WithContentType sets the Content-Type header for the request. +// This function will overwrite any existing Content-Type header. func WithContentType(ct string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set("Content-Type", ct) @@ -1161,6 +1202,8 @@ func WithContentType(ct string) RequestOpt { } } +// WithUserAgent sets the User-Agent header for the request. +// This function will overwrite any existing User-Agent header. func WithUserAgent(ua string) RequestOpt { return func(opt *RequestOpts) error { opt.headers.Set("User-Agent", ua) @@ -1168,6 +1211,8 @@ func WithUserAgent(ua string) RequestOpt { } } +// WithSkipTLSVerify sets whether the request will skip TLS verification. +// If set to true, the request will not verify the server's TLS certificate. func WithSkipTLSVerify(skip bool) RequestOpt { return func(opt *RequestOpts) error { opt.skipTLSVerify = skip @@ -1175,13 +1220,23 @@ func WithSkipTLSVerify(skip bool) RequestOpt { } } +/* +// WithDisableRedirect sets whether the request will disable HTTP redirects. +// If set to true, the request will not follow redirects automatically. +// For example, if the server responds with a 301 or 302 status code, the request will not automatically follow the redirect. +// You will get the original response with the redirect status code and Location header. func WithDisableRedirect(disable bool) RequestOpt { return func(opt *RequestOpts) error { opt.disableRedirect = disable return nil } } +*/ +// WithDoRawRequest sets whether the request will be sent as a raw request without any modifications. +// You can use this with function SetRawRequest to set a custom http.Request. +// If set to true, the request will not apply any modifications to the request headers, body, or other settings. +// If set to false, the request will apply the modifications as usual. func WithDoRawRequest(doRawRequest bool) RequestOpt { return func(opt *RequestOpts) error { opt.doRawRequest = doRawRequest @@ -1189,27 +1244,16 @@ func WithDoRawRequest(doRawRequest bool) RequestOpt { } } -func WithDoRawClient(doRawClient bool) RequestOpt { - return func(opt *RequestOpts) error { - opt.doRawClient = doRawClient - return nil - } -} - -func WithDoRawTransport(doRawTrans bool) RequestOpt { - return func(opt *RequestOpts) error { - opt.doRawTransport = doRawTrans - return nil - } -} - +// WithTransport sets the http.Transport used for the request. func WithTransport(hs *http.Transport) RequestOpt { return func(opt *RequestOpts) error { opt.transport = hs + opt.customTransport = true return nil } } +// WithRawRequest sets a custom http.Request for the request. func WithRawRequest(req *http.Request) RequestOpt { return func(opt *RequestOpts) error { opt.rawRequest = req @@ -1217,6 +1261,7 @@ func WithRawRequest(req *http.Request) RequestOpt { } } +// WithRawClient sets a custom http.Client for the request. func WithRawClient(hc *http.Client) RequestOpt { return func(opt *RequestOpts) error { opt.rawClient = hc @@ -1224,6 +1269,9 @@ func WithRawClient(hc *http.Client) RequestOpt { } } +// WithCustomHostIP sets custom IPs for the host. +// it means that the request will use the specified IPs to resolve the host instead of using DNS. +// Note: LookUpIPfn will be ignored if this function is used. func WithCustomHostIP(ip []string) RequestOpt { return func(opt *RequestOpts) error { if len(ip) == 0 { @@ -1239,6 +1287,7 @@ func WithCustomHostIP(ip []string) RequestOpt { } } +// WithAddCustomHostIP adds a custom IP to the request. func WithAddCustomHostIP(ip string) RequestOpt { return func(opt *RequestOpts) error { if net.ParseIP(ip) == nil { @@ -1249,6 +1298,9 @@ func WithAddCustomHostIP(ip string) RequestOpt { } } +// WithLookUpFn sets a custom function to look up IP addresses for the host. +// If set to nil, it will use the default net.Resolver.LookupIPAddr function. +// Note: If customDNS is set, this function will not be used. func WithLookUpFn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) RequestOpt { return func(opt *RequestOpts) error { if lookUpIPfn == nil { @@ -1327,23 +1379,29 @@ func (b *Body) readAll() { } } +// String will read the body and return it as a string. +// if the body is too large, it may cause high memory usage. func (b *Body) String() string { b.readAll() return string(b.full) } +// Bytes will read the body and return it as a byte slice. +// if the body is too large, it may cause high memory usage. func (b *Body) Bytes() []byte { b.readAll() return b.full } +// Unmarshal will read the body and unmarshal it into the given interface using json.Unmarshal +// if the body is too large, it may cause high memory usage. func (b *Body) Unmarshal(u interface{}) error { b.readAll() return json.Unmarshal(b.full, u) } // Reader returns a reader for the body -// if this function is called, other functions like String, Bytes, Unmarshal may not work +// if this function is called, other functions like String, Bytes, Unmarshal not work func (b *Body) Reader() io.ReadCloser { b.Lock() defer b.Unlock() @@ -1354,18 +1412,22 @@ func (b *Body) Reader() io.ReadCloser { return b.raw } +// Close closes the body reader. func (b *Body) Close() error { return b.raw.Close() } +// GetRequest returns the original Request object associated with the Response. func (r *Response) GetRequest() Request { return r.req } +// Body returns the Body object associated with the Response. func (r *Response) Body() *Body { return r.data } +// Close closes the response body and releases any resources associated with it. func (r *Response) Close() error { if r != nil && r.data != nil && r.data.raw != nil { return r.Response.Body.Close() @@ -1373,6 +1435,8 @@ func (r *Response) Close() error { return nil } +// CloseAll closes the response body and releases any resources associated with it. +// It also closes all idle connections in the http.Client if it is not nil. func (r *Response) CloseAll() error { if r.rawClient != nil { r.rawClient.CloseIdleConnections() @@ -1380,16 +1444,19 @@ func (r *Response) CloseAll() error { return r.Close() } +// HttpClient returns the http.Client used for the request. func (r *Response) HttpClient() *http.Client { return r.rawClient } +// Curl sends the HTTP request and returns the response. func Curl(r *Request) (*Response, error) { r.errInfo = nil err := applyOptions(r) if err != nil { return nil, fmt.Errorf("apply options error: %s", err) } + r.rawRequest = r.rawRequest.WithContext(r.doCtx) resp, err := r.rawClient.Do(r.rawRequest) var res = Response{ Response: resp, @@ -1410,43 +1477,38 @@ func Curl(r *Request) (*Response, error) { return &res, r.errInfo } +// NewReq creates a new Request with the specified URI and default method "GET". func NewReq(uri string, opts ...RequestOpt) *Request { return NewSimpleRequest(uri, "GET", opts...) } +// NewReqWithContext creates a new Request with the specified URI and default method "GET" using the provided context. func NewReqWithContext(ctx context.Context, uri string, opts ...RequestOpt) *Request { return NewSimpleRequestWithContext(ctx, uri, "GET", opts...) } +// NewSimpleRequest creates a new Request with the specified URI and method. func NewSimpleRequest(uri string, method string, opts ...RequestOpt) *Request { r, _ := newRequest(context.Background(), uri, method, opts...) return r } +// NewRequest creates a new Request with the specified URI and method. func NewRequest(uri string, method string, opts ...RequestOpt) (*Request, error) { return newRequest(context.Background(), uri, method, opts...) } +// NewSimpleRequestWithContext creates a new Request with the specified URI and method using the provided context. func NewSimpleRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) *Request { r, _ := newRequest(ctx, uri, method, opts...) return r } +// NewRequestWithContext creates a new Request with the specified URI and method using the provided context. func NewRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) { return newRequest(ctx, uri, method, opts...) } -func NewHttpClient(opts ...RequestOpt) (*http.Client, error) { - req, err := newRequest(context.Background(), "", "", opts...) - if err != nil { - return nil, err - } - defer func() { - req = nil - }() - return req.HttpClient() -} - func newRequest(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) { var req *http.Request var err error @@ -1487,63 +1549,21 @@ func newRequest(ctx context.Context, uri string, method string, opts ...RequestO } } } - if r.transport == nil && !r.doRawTransport { - r.transport = &http.Transport{} - } - if r.rawClient == nil && !r.doRawClient { + if r.rawClient == nil { r.rawClient = new(http.Client) } - if !r.doRawTransport { - if r.skipTLSVerify { - if r.transport.TLSClientConfig == nil { - r.transport.TLSClientConfig = &tls.Config{} - } - r.transport.TLSClientConfig.InsecureSkipVerify = true + if r.tlsConfig == nil { + r.tlsConfig = &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, } - if r.tlsConfig != nil { - r.transport.TLSClientConfig = r.tlsConfig - } - r.transport.DialContext = func(ctx context.Context, netType, addr string) (net.Conn, error) { - var lastErr error - var addrs []string - host, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - if len(r.customIP) > 0 { - for _, v := range r.customIP { - ipAddr := net.ParseIP(v) - if ipAddr == nil { - return nil, fmt.Errorf("invalid custom ip: %s", r.customIP) - } - tmpAddr := net.JoinHostPort(v, port) - addrs = append(addrs, tmpAddr) - } - } else { - ipLists, err := r.lookUpIPfn(ctx, host) - if err != nil { - return nil, err - } - for _, v := range ipLists { - tmpAddr := net.JoinHostPort(v.String(), port) - addrs = append(addrs, tmpAddr) - } - } - for _, addr := range addrs { - c, err := net.DialTimeout(netType, addr, r.dialTimeout) - if err != nil { - lastErr = err - continue - } - if r.timeout != 0 { - err = c.SetDeadline(time.Now().Add(r.timeout)) - } - return c, nil - } - return nil, lastErr - } - if r.dialFn != nil { - r.transport.DialContext = r.dialFn + } + r.tlsConfig.InsecureSkipVerify = r.skipTLSVerify + if r.transport == nil { + r.transport = &http.Transport{ + ForceAttemptHTTP2: true, + DialContext: DefaultDialFunc, + DialTLSContext: DefaultDialTlsFunc, + Proxy: DefaultProxyURL(), } } return r, nil @@ -1610,7 +1630,7 @@ func applyDataReader(r *Request) error { return } } - if _, err := copyWithContext(r.ctx, r.FileUploadRecallFn, v.FileName, v.FileSize, fw, v.FileData); err != nil { + if _, err := copyWithContext(r.ctx, r.fileUploadRecallFn, v.FileName, v.FileSize, fw, v.FileData); err != nil { r.errInfo = err r.bodyFileData[idx] = v pw.CloseWithError(err) // close pipe with error @@ -1672,51 +1692,52 @@ func applyOptions(r *Request) error { } } } - if r.proxy != "" { - purl, err := url.Parse(r.proxy) - if err != nil { - return fmt.Errorf("parse proxy url error: %s", err) - } - r.transport.Proxy = http.ProxyURL(purl) - } - if !r.doRawClient { - if !r.doRawTransport { - if !r.alreadySetLookUpIPfn && len(r.customIP) > 0 { - resolver := net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { - for _, addr := range r.customIP { - if conn, err = net.Dial("udp", addr+":53"); err != nil { - continue - } else { - return conn, nil - } - } - return - }, + if !r.alreadySetLookUpIPfn && len(r.customDNS) > 0 { + resolver := net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { + for _, addr := range r.customDNS { + if conn, err = net.Dial("udp", addr+":53"); err != nil { + continue + } else { + return conn, nil + } } - r.lookUpIPfn = resolver.LookupIPAddr - } + return + }, } - if r.skipTLSVerify { - if r.transport.TLSClientConfig == nil { - r.transport.TLSClientConfig = &tls.Config{} - } - r.transport.TLSClientConfig.InsecureSkipVerify = true - } - if r.tlsConfig != nil { - r.transport.TLSClientConfig = r.tlsConfig - } - if r.dialFn != nil { - r.transport.DialContext = r.dialFn - } - r.rawClient.Transport = r.transport - if r.disableRedirect { - r.rawClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } + r.lookUpIPfn = resolver.LookupIPAddr + } + if r.tlsConfig == nil { + r.tlsConfig = &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, } } + if r.tlsConfig.ServerName == "" { + r.tlsConfig.ServerName = r.rawRequest.URL.Hostname() + } + + r.tlsConfig.InsecureSkipVerify = r.skipTLSVerify + + if r.rawClient.Transport == nil { + r.rawClient.Transport = &Transport{base: r.transport} + } + + r.doCtx = context.WithValue(context.WithValue(r.ctx, "dialTimeout", r.dialTimeout), "timeout", r.timeout) + r.doCtx = context.WithValue(r.doCtx, "lookUpIP", r.lookUpIPfn) + if r.customIP != nil && len(r.customIP) > 0 { + r.doCtx = context.WithValue(r.doCtx, "customIP", r.customIP) + } + r.doCtx = context.WithValue(r.doCtx, "tlsConfig", r.tlsConfig) + if r.proxy != "" { + r.doCtx = context.WithValue(r.doCtx, "proxy", r.proxy) + } + if r.dialFn != nil { + r.doCtx = context.WithValue(r.doCtx, "dialFn", r.dialFn) + } + if r.customTransport { + r.doCtx = context.WithValue(r.doCtx, "custom", r.transport) + } return nil } @@ -1764,53 +1785,33 @@ func copyWithContext(ctx context.Context, recall func(string, int64, int64), fil } } -func NewReqWithClient(client *http.Client, uri string, opts ...RequestOpt) *Request { +func NewReqWithClient(client Client, uri string, opts ...RequestOpt) *Request { return NewSimpleRequestWithClient(client, uri, "GET", opts...) } -func NewReqWithContextWithClient(ctx context.Context, client *http.Client, uri string, opts ...RequestOpt) *Request { +func NewReqWithContextWithClient(ctx context.Context, client Client, uri string, opts ...RequestOpt) *Request { return NewSimpleRequestWithContextWithClient(ctx, client, uri, "GET", opts...) } -func NewSimpleRequestWithClient(client *http.Client, uri string, method string, opts ...RequestOpt) *Request { +func NewSimpleRequestWithClient(client Client, uri string, method string, opts ...RequestOpt) *Request { r, _ := NewRequestWithContextWithClient(context.Background(), client, uri, method, opts...) return r } -func NewRequestWithClient(client *http.Client, uri string, method string, opts ...RequestOpt) (*Request, error) { +func NewRequestWithClient(client Client, uri string, method string, opts ...RequestOpt) (*Request, error) { return NewRequestWithContextWithClient(context.Background(), client, uri, method, opts...) } -func NewSimpleRequestWithContextWithClient(ctx context.Context, client *http.Client, uri string, method string, opts ...RequestOpt) *Request { +func NewSimpleRequestWithContextWithClient(ctx context.Context, client Client, uri string, method string, opts ...RequestOpt) *Request { r, _ := NewRequestWithContextWithClient(ctx, client, uri, method, opts...) return r } -func NewRequestWithContextWithClient(ctx context.Context, client *http.Client, uri string, method string, opts ...RequestOpt) (*Request, error) { +func NewRequestWithContextWithClient(ctx context.Context, client Client, uri string, method string, opts ...RequestOpt) (*Request, error) { req, err := newRequest(ctx, uri, method, opts...) if err != nil { return nil, err } - req.rawClient = client - req.SetDoRawClient(true) - req.SetDoRawTransport(true) + req.rawClient = client.Client return req, err } - -func SetClientSkipVerify(c *http.Client, val bool) error { - switch tp := c.Transport.(type) { - case *http.Transport: - if tp.TLSClientConfig == nil { - tp.TLSClientConfig = &tls.Config{} - } - tp.TLSClientConfig.InsecureSkipVerify = val - case nil: - transport := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: val}, - } - c.Transport = transport - default: - return fmt.Errorf("unsupported transport type: %T", tp) - } - return nil -} diff --git a/curl_default.go b/curl_default.go new file mode 100644 index 0000000..5c14c6c --- /dev/null +++ b/curl_default.go @@ -0,0 +1,197 @@ +package starnet + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded` + HEADER_FORM_DATA = `multipart/form-data` + HEADER_JSON = `application/json` + HEADER_PLAIN = `text/plain` +) + +var ( + DefaultDialTimeout = 5 * time.Second + DefaultTimeout = 10 * time.Second + DefaultFetchRespBody = false +) + +func UrlEncodeRaw(str string) string { + strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1) + return strs +} + +func UrlEncode(str string) string { + return url.QueryEscape(str) +} + +func UrlDecode(str string) (string, error) { + return url.QueryUnescape(str) +} + +func BuildQuery(queryData map[string]string) string { + query := url.Values{} + for k, v := range queryData { + query.Add(k, v) + } + return query.Encode() +} + +// BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string, +// and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests, +// where the server expects the request body to be URL-encoded form data. +// +// Parameters: +// queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request. +// +// Returns: +// A byte slice representing the URL-encoded form data. +func BuildPostForm(queryMap map[string]string) []byte { + return []byte(BuildQuery(queryMap)) +} + +func Get(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "GET", opts...).Do() +} + +func Post(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "POST", opts...).Do() +} + +func Options(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "OPTIONS", opts...).Do() +} + +func Put(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "PUT", opts...).Do() +} + +func Delete(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "DELETE", opts...).Do() +} + +func Head(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "HEAD", opts...).Do() +} + +func Patch(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "PATCH", opts...).Do() +} + +func Trace(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "TRACE", opts...).Do() +} + +func Connect(uri string, opts ...RequestOpt) (*Response, error) { + return NewSimpleRequest(uri, "CONNECT", opts...).Do() +} + +func DefaultCheckRedirectFunc(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +func DefaultDialFunc(ctx context.Context, netType, addr string) (net.Conn, error) { + var lastErr error + var addrs []string + if dialFn, ok := ctx.Value("dialFunc").(func(context.Context, string, string) (net.Conn, error)); ok { + if dialFn != nil { + return dialFn(ctx, netType, addr) + } + } + customIP, ok := ctx.Value("customIP").([]string) + if !ok { + customIP = nil + } + dialTimeout, ok := ctx.Value("dialTimeout").(time.Duration) + if !ok { + dialTimeout = DefaultDialTimeout + } + timeout, ok := ctx.Value("timeout").(time.Duration) + if !ok { + timeout = DefaultTimeout + } + lookUpIPfn, ok := ctx.Value("lookUpIP").(func(context.Context, string) ([]net.IPAddr, error)) + if !ok { + lookUpIPfn = net.DefaultResolver.LookupIPAddr + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + proxy, ok := ctx.Value("proxy").(string) + if !ok { + proxy = "" + } + if proxy == "" && len(customIP) > 0 { + for _, v := range customIP { + ipAddr := net.ParseIP(v) + if ipAddr == nil { + return nil, fmt.Errorf("invalid custom ip: %s", customIP) + } + tmpAddr := net.JoinHostPort(v, port) + addrs = append(addrs, tmpAddr) + } + } else { + ipLists, err := lookUpIPfn(ctx, host) + if err != nil { + return nil, err + } + for _, v := range ipLists { + tmpAddr := net.JoinHostPort(v.String(), port) + addrs = append(addrs, tmpAddr) + } + } + for _, addr := range addrs { + c, err := net.DialTimeout(netType, addr, dialTimeout) + if err != nil { + lastErr = err + continue + } + if timeout != 0 { + err = c.SetDeadline(time.Now().Add(timeout)) + } + return c, nil + } + return nil, lastErr +} + +func DefaultDialTlsFunc(ctx context.Context, netType, addr string) (net.Conn, error) { + conn, err := DefaultDialFunc(ctx, netType, addr) + if err != nil { + return nil, err + } + tlsConfig, ok := ctx.Value("tlsConfig").(*tls.Config) + if !ok || tlsConfig == nil { + return nil, fmt.Errorf("tlsConfig is not set in context") + } + tlsConn := tls.Client(conn, tlsConfig) + if err := tlsConn.Handshake(); err != nil { + return nil, fmt.Errorf("tls handshake failed: %w", err) + } + return tlsConn, nil +} + +func DefaultProxyURL() func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + if req == nil { + return nil, fmt.Errorf("request is nil") + } + proxyURL, ok := req.Context().Value("proxy").(string) + if !ok || proxyURL == "" { + return nil, nil + } + parsedURL, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy URL: %w", err) + } + return parsedURL, nil + } +} diff --git a/curl_test.go b/curl_test.go index c06266b..37baec3 100644 --- a/curl_test.go +++ b/curl_test.go @@ -345,11 +345,11 @@ func TestGet(t *testing.T) { var reply postmanReply resp, err := NewReq("https://postman-echo.com/get"). AddHeader("hello", "nononmo"). - SetAutoCalcContentLength(true). - Do() + SetAutoCalcContentLengthNoError(true).Do() if err != nil { t.Error(err) } + fmt.Println(resp.Proto) err = resp.Body().Unmarshal(&reply) if err != nil { t.Error(err) @@ -474,7 +474,7 @@ func TestReqClone(t *testing.T) { })) defer server.Close() - req := NewSimpleRequestWithClient(http.DefaultClient, server.URL, "GET", WithHeader("hello", "world")) + req := NewSimpleRequestWithClient(NewClientFromHttpClientNoError(http.DefaultClient), server.URL, "GET", WithHeader("hello", "world")) resp, err := req.Do() if err != nil { t.Error(err) @@ -526,7 +526,7 @@ func TestUploadFile(t *testing.T) { })) defer server.Close() - req := NewSimpleRequestWithClient(http.DefaultClient, server.URL, "GET", WithHeader("hello", "world")) + req := NewSimpleRequestWithClient(NewClientFromHttpClientNoError(http.DefaultClient), server.URL, "GET", WithHeader("hello", "world")) req.AddFileWithName("666", "./curl.go", "curl.go") req.AddFile("777", "./go.mod") req.AddFileWithNameAndType("888", "./ping.go", "ping.go", "html") @@ -569,13 +569,15 @@ func TestTlsConfig(t *testing.T) { } req := NewSimpleRequestWithClient(client, server.URL, "GET", WithHeader("hello", "world")) //SetClientSkipVerify(client, true) - req.SetDoRawClient(false) + //req.SetDoRawClient(false) //req.SetDoRawTransport(false) req.SetSkipTLSVerify(true) + req.SetProxy("http://127.0.0.1:29992") resp, err := req.Do() if err != nil { t.Error(err) } + fmt.Println(resp.Proto) if resp.StatusCode != 200 { resp.CloseAll() t.Errorf("status code is %d", resp.StatusCode) @@ -592,4 +594,10 @@ func TestTlsConfig(t *testing.T) { t.Errorf("status code is %d", resp.StatusCode) } resp.CloseAll() + req = req.Clone() + req.SetSkipTLSVerify(false) + resp, err = req.Do() + if err == nil { + t.Error(err) + } } diff --git a/curl_transport.go b/curl_transport.go new file mode 100644 index 0000000..08ff99b --- /dev/null +++ b/curl_transport.go @@ -0,0 +1,134 @@ +package starnet + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "reflect" +) + +type Client struct { + *http.Client +} + +// NewHttpClient creates a new http.Client with the specified options. +func NewHttpClient(opts ...RequestOpt) (Client, error) { + req, err := newRequest(context.Background(), "", "", opts...) + if err != nil { + return Client{}, err + } + defer func() { + req = nil + }() + cl, err := req.HttpClient() + return Client{ + Client: cl, + }, err +} + +func NewClientFromHttpClient(httpClient *http.Client) (Client, error) { + if httpClient == nil { + return Client{}, fmt.Errorf("httpClient cannot be nil") + } + + if httpClient.Transport == nil { + httpClient.Transport = &Transport{ + base: &http.Transport{}, + } + } else { + switch t := httpClient.Transport.(type) { + case *Transport: + if t.base == nil { + t.base = &http.Transport{} + } + case *http.Transport: + httpClient.Transport = &Transport{ + base: t, + } + default: + return Client{}, fmt.Errorf("unsupported transport type: %T", t) + } + } + return Client{ + Client: httpClient, + }, nil +} + +func NewClientFromHttpClientNoError(httpClient *http.Client) Client { + return Client{Client: httpClient} +} + +// DisableRedirect returns whether the request will disable HTTP redirects. +// if true, the request will not follow redirects automatically. +// for example, if the server responds with a 301 or 302 status code, the request will not automatically follow the redirect. +// you will get the original response with the redirect status code and Location header. +func (c Client) DisableRedirect() bool { + return reflect.ValueOf(c.Client.CheckRedirect).Pointer() == reflect.ValueOf(DefaultCheckRedirectFunc).Pointer() +} + +// SetDisableRedirect sets whether the request will disable HTTP redirects. +// if true, the request will not follow redirects automatically. +// for example, if the server responds with a 301 or 302 status code, the request will not automatically follow the redirect. +// you will get the original response with the redirect status code and Location header. +func (c Client) SetDisableRedirect(disableRedirect bool) { + if disableRedirect { + c.Client.CheckRedirect = DefaultCheckRedirectFunc + } +} + +func (c Client) SetDefaultSkipTLSVerify(skip bool) { + if c.Client.Transport == nil { + c.Client.Transport = &Transport{ + base: &http.Transport{}, + } + } + if transport, ok := c.Client.Transport.(*Transport); ok { + if transport.base.TLSClientConfig == nil { + transport.base.TLSClientConfig = &tls.Config{} + } + transport.base.TLSClientConfig.InsecureSkipVerify = skip + } else if transport, ok := c.Client.Transport.(*http.Transport); ok { + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = skip + } +} + +func (c Client) SetDefaultTLSConfig(tlsConfig *tls.Config) { + if c.Client.Transport == nil { + c.Client.Transport = &Transport{ + base: &http.Transport{}, + } + } + if transport, ok := c.Client.Transport.(*Transport); ok { + transport.base.TLSClientConfig = tlsConfig + } else if transport, ok := c.Client.Transport.(*http.Transport); ok { + transport.TLSClientConfig = tlsConfig + } +} + +type Transport struct { + base *http.Transport +} + +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.base == nil { + t.base = &http.Transport{} + } + transport, ok := req.Context().Value("transport").(*http.Transport) + if ok && transport != nil { + return transport.RoundTrip(req) + } + proxy, ok := req.Context().Value("proxy").(string) + if ok && proxy != "" { + tlsConfig, ok := req.Context().Value("tlsConfig").(*tls.Config) + if ok && tlsConfig != nil { + tmpTransport := t.base.Clone() + tmpTransport.TLSClientConfig = tlsConfig + return tmpTransport.RoundTrip(req) + } + } + return t.base.RoundTrip(req) +}