Compare commits

...

6 Commits

Author SHA1 Message Date
b90c59d6e7
修改版本号 2025-08-21 21:40:29 +08:00
4e154cc17b
update benchmark 2025-08-21 21:37:21 +08:00
67b0025f9c
更新content-length的默认处理方式 2025-08-21 19:17:19 +08:00
c4fa62536a
为client新增部分函数 2025-08-21 15:32:19 +08:00
260ceb90ed
重构http Client部分 2025-08-21 15:02:02 +08:00
d260181adf
update 2025-08-15 15:07:51 +08:00
5 changed files with 1172 additions and 397 deletions

854
curl.go

File diff suppressed because it is too large Load Diff

198
curl_default.go Normal file
View File

@ -0,0 +1,198 @@
package starnet
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"net/url"
"strings"
"time"
)
const (
HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded`
HEADER_FORM_DATA = `multipart/form-data`
HEADER_JSON = `application/json`
HEADER_PLAIN = `text/plain`
)
var (
DefaultDialTimeout = 5 * time.Second
DefaultTimeout = 10 * time.Second
DefaultFetchRespBody = false
DefaultHttpClient = NewHttpClientNoErr()
)
func UrlEncodeRaw(str string) string {
strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1)
return strs
}
func UrlEncode(str string) string {
return url.QueryEscape(str)
}
func UrlDecode(str string) (string, error) {
return url.QueryUnescape(str)
}
func BuildQuery(queryData map[string]string) string {
query := url.Values{}
for k, v := range queryData {
query.Add(k, v)
}
return query.Encode()
}
// BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string,
// and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests,
// where the server expects the request body to be URL-encoded form data.
//
// Parameters:
// queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request.
//
// Returns:
// A byte slice representing the URL-encoded form data.
func BuildPostForm(queryMap map[string]string) []byte {
return []byte(BuildQuery(queryMap))
}
func Get(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "GET", opts...).Do()
}
func Post(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "POST", opts...).Do()
}
func Options(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "OPTIONS", opts...).Do()
}
func Put(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "PUT", opts...).Do()
}
func Delete(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "DELETE", opts...).Do()
}
func Head(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "HEAD", opts...).Do()
}
func Patch(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "PATCH", opts...).Do()
}
func Trace(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "TRACE", opts...).Do()
}
func Connect(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequestWithClient(DefaultHttpClient, uri, "CONNECT", opts...).Do()
}
func DefaultCheckRedirectFunc(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
func DefaultDialFunc(ctx context.Context, netType, addr string) (net.Conn, error) {
var lastErr error
var addrs []string
if dialFn, ok := ctx.Value("dialFunc").(func(context.Context, string, string) (net.Conn, error)); ok {
if dialFn != nil {
return dialFn(ctx, netType, addr)
}
}
customIP, ok := ctx.Value("customIP").([]string)
if !ok {
customIP = nil
}
dialTimeout, ok := ctx.Value("dialTimeout").(time.Duration)
if !ok {
dialTimeout = DefaultDialTimeout
}
timeout, ok := ctx.Value("timeout").(time.Duration)
if !ok {
timeout = DefaultTimeout
}
lookUpIPfn, ok := ctx.Value("lookUpIP").(func(context.Context, string) ([]net.IPAddr, error))
if !ok {
lookUpIPfn = net.DefaultResolver.LookupIPAddr
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
proxy, ok := ctx.Value("proxy").(string)
if !ok {
proxy = ""
}
if proxy == "" && len(customIP) > 0 {
for _, v := range customIP {
ipAddr := net.ParseIP(v)
if ipAddr == nil {
return nil, fmt.Errorf("invalid custom ip: %s", customIP)
}
tmpAddr := net.JoinHostPort(v, port)
addrs = append(addrs, tmpAddr)
}
} else {
ipLists, err := lookUpIPfn(ctx, host)
if err != nil {
return nil, err
}
for _, v := range ipLists {
tmpAddr := net.JoinHostPort(v.String(), port)
addrs = append(addrs, tmpAddr)
}
}
for _, addr := range addrs {
c, err := net.DialTimeout(netType, addr, dialTimeout)
if err != nil {
lastErr = err
continue
}
if timeout != 0 {
err = c.SetDeadline(time.Now().Add(timeout))
}
return c, nil
}
return nil, lastErr
}
func DefaultDialTlsFunc(ctx context.Context, netType, addr string) (net.Conn, error) {
conn, err := DefaultDialFunc(ctx, netType, addr)
if err != nil {
return nil, err
}
tlsConfig, ok := ctx.Value("tlsConfig").(*tls.Config)
if !ok || tlsConfig == nil {
return nil, fmt.Errorf("tlsConfig is not set in context")
}
tlsConn := tls.Client(conn, tlsConfig)
if err := tlsConn.Handshake(); err != nil {
return nil, fmt.Errorf("tls handshake failed: %w", err)
}
return tlsConn, nil
}
func DefaultProxyURL() func(*http.Request) (*url.URL, error) {
return func(req *http.Request) (*url.URL, error) {
if req == nil {
return nil, fmt.Errorf("request is nil")
}
proxyURL, ok := req.Context().Value("proxy").(string)
if !ok || proxyURL == "" {
return nil, nil
}
parsedURL, err := url.Parse(proxyURL)
if err != nil {
return nil, fmt.Errorf("failed to parse proxy URL: %w", err)
}
return parsedURL, nil
}
}

View File

@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestUrlEncodeRaw(t *testing.T) {
@ -89,7 +90,7 @@ func TestGetRequest(t *testing.T) {
}))
defer server.Close()
resp, err := Get(server.URL)
resp, err := Get(server.URL, WithSkipTLSVerify(true), WithHeader("hello", "world"), WithUserAgent("hello world"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
@ -345,11 +346,11 @@ func TestGet(t *testing.T) {
var reply postmanReply
resp, err := NewReq("https://postman-echo.com/get").
AddHeader("hello", "nononmo").
SetAutoCalcContentLength(true).
Do()
SetAutoCalcContentLengthNoError(true).Do()
if err != nil {
t.Error(err)
}
fmt.Println(resp.Proto)
err = resp.Body().Unmarshal(&reply)
if err != nil {
t.Error(err)
@ -474,7 +475,7 @@ func TestReqClone(t *testing.T) {
}))
defer server.Close()
req := NewSimpleRequestWithClient(http.DefaultClient, server.URL, "GET", WithHeader("hello", "world"))
req := NewSimpleRequestWithClient(NewClientFromHttpClientNoError(http.DefaultClient), server.URL, "GET", WithHeader("hello", "world"))
resp, err := req.Do()
if err != nil {
t.Error(err)
@ -526,7 +527,7 @@ func TestUploadFile(t *testing.T) {
}))
defer server.Close()
req := NewSimpleRequestWithClient(http.DefaultClient, server.URL, "GET", WithHeader("hello", "world"))
req := NewSimpleRequestWithClient(NewClientFromHttpClientNoError(http.DefaultClient), server.URL, "GET", WithHeader("hello", "world"))
req.AddFileWithName("666", "./curl.go", "curl.go")
req.AddFile("777", "./go.mod")
req.AddFileWithNameAndType("888", "./ping.go", "ping.go", "html")
@ -552,3 +553,146 @@ func TestUploadFile(t *testing.T) {
}
resp.CloseAll()
}
func TestTlsConfig(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
if req.Header.Get("hello") != "world" {
rw.WriteHeader(http.StatusBadRequest)
rw.Write([]byte("hello world failed"))
return
}
rw.Write([]byte(`OK`))
}))
defer server.Close()
client, err := NewHttpClient(WithSkipTLSVerify(false))
if err != nil {
t.Error(err)
}
req := client.NewSimpleRequest(server.URL, "GET", WithHeader("hello", "world"))
//SetClientSkipVerify(client, true)
//req.SetDoRawClient(false)
//req.SetDoRawTransport(false)
req.SetSkipTLSVerify(true)
req.SetProxy("http://127.0.0.1:29992")
resp, err := req.Do()
if err != nil {
t.Error(err)
}
fmt.Println(resp.Proto)
if resp.StatusCode != 200 {
resp.CloseAll()
t.Errorf("status code is %d", resp.StatusCode)
}
resp.CloseAll()
req = req.Clone()
req.AddHeader("ok", "good")
resp, err = req.Do()
if err != nil {
t.Error(err)
}
if resp.StatusCode != 200 {
resp.CloseAll()
t.Errorf("status code is %d", resp.StatusCode)
}
resp.CloseAll()
req = req.Clone()
req.SetSkipTLSVerify(false)
resp, err = req.Do()
if err == nil {
t.Error(err)
}
}
func TestHttpPostAndChunked(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodPost {
t.Errorf("Expected 'POST', got %v", req.Method)
}
buf := make([]byte, 1024)
n, _ := req.Body.Read(buf)
if string(buf[:n]) != "hello world" {
t.Errorf("Expected body to be 'hello world', got %s", string(buf[:n]))
}
if req.Header.Get("chunked") == "true" {
if req.TransferEncoding[0] != "chunked" {
t.Errorf("Expected Transfer-Encoding to be 'chunked', got %s", req.Header.Get("Transfer-Encoding"))
}
} else {
if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" {
t.Errorf("Expected Transfer-Encoding to not be 'chunked', got %s", req.Header.Get("Transfer-Encoding"))
}
}
rw.Write([]byte(`OK`))
}))
defer server.Close()
resp, err := Post(server.URL, WithBytes([]byte("hello world")), WithContentLength(-1), WithHeader("Content-Type", "text/plain"),
WithHeader("chunked", "true"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != "OK" {
t.Errorf("Expected OK, got %v", body)
}
resp.Close()
resp, err = Post(server.URL, WithBytes([]byte("hello world")), WithHeader("Content-Type", "text/plain"),
WithHeader("chunked", "false"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
defer resp.Close()
body = resp.Body().String()
if body != "OK" {
t.Errorf("Expected OK, got %v", body)
}
}
func TestWithTimeout(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
time.Sleep(time.Second * 30)
rw.Write([]byte(`OK`))
}))
funcList := []func(string, ...RequestOpt) (*Response, error){
Get,
Post,
Put,
Delete,
Options,
Patch,
Head,
Trace,
Connect,
}
defer server.Close()
for i := 1; i < 30; i++ {
go func(i int) {
old := time.Now()
fn := funcList[i%len(funcList)]
resp, err := fn(server.URL, WithTimeout(time.Second*time.Duration(i)))
if time.Since(old) > time.Second*time.Duration(i+2) || time.Since(old) < time.Second*time.Duration(i) {
t.Errorf("timeout not work")
}
fmt.Println(time.Since(old))
if err == nil {
t.Error(err)
resp.CloseAll()
} else {
fmt.Println(err)
}
}(i)
}
resp, err := Get(server.URL, WithTimeout(time.Second*60))
if err != nil {
t.Error(err)
} else {
fmt.Println(resp.Body().String())
if resp.StatusCode != 200 {
resp.CloseAll()
t.Errorf("status code is %d", resp.StatusCode)
}
resp.CloseAll()
}
}

165
curl_transport.go Normal file
View File

@ -0,0 +1,165 @@
package starnet
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"reflect"
)
type Client struct {
*http.Client
}
// NewHttpClient creates a new http.Client with the specified options.
func NewHttpClient(opts ...RequestOpt) (Client, error) {
req, err := newRequest(context.Background(), "", "", opts...)
if err != nil {
return Client{}, err
}
defer func() {
req = nil
}()
cl, err := req.HttpClient()
return Client{
Client: cl,
}, err
}
func NewHttpClientNoErr(opts ...RequestOpt) Client {
c, _ := NewHttpClient(opts...)
return c
}
func NewClientFromHttpClient(httpClient *http.Client) (Client, error) {
if httpClient == nil {
return Client{}, fmt.Errorf("httpClient cannot be nil")
}
if httpClient.Transport == nil {
httpClient.Transport = &Transport{
base: &http.Transport{},
}
} else {
switch t := httpClient.Transport.(type) {
case *Transport:
if t.base == nil {
t.base = &http.Transport{}
}
case *http.Transport:
httpClient.Transport = &Transport{
base: t,
}
default:
return Client{}, fmt.Errorf("unsupported transport type: %T", t)
}
}
return Client{
Client: httpClient,
}, nil
}
func NewClientFromHttpClientNoError(httpClient *http.Client) Client {
return Client{Client: httpClient}
}
// DisableRedirect returns whether the request will disable HTTP redirects.
// if true, the request will not follow redirects automatically.
// for example, if the server responds with a 301 or 302 status code, the request will not automatically follow the redirect.
// you will get the original response with the redirect status code and Location header.
func (c Client) DisableRedirect() bool {
return reflect.ValueOf(c.Client.CheckRedirect).Pointer() == reflect.ValueOf(DefaultCheckRedirectFunc).Pointer()
}
// SetDisableRedirect sets whether the request will disable HTTP redirects.
// if true, the request will not follow redirects automatically.
// for example, if the server responds with a 301 or 302 status code, the request will not automatically follow the redirect.
// you will get the original response with the redirect status code and Location header.
func (c Client) SetDisableRedirect(disableRedirect bool) {
if disableRedirect {
c.Client.CheckRedirect = DefaultCheckRedirectFunc
}
}
func (c Client) SetDefaultSkipTLSVerify(skip bool) {
if c.Client.Transport == nil {
c.Client.Transport = &Transport{
base: &http.Transport{},
}
}
if transport, ok := c.Client.Transport.(*Transport); ok {
if transport.base.TLSClientConfig == nil {
transport.base.TLSClientConfig = &tls.Config{}
}
transport.base.TLSClientConfig.InsecureSkipVerify = skip
} else if transport, ok := c.Client.Transport.(*http.Transport); ok {
if transport.TLSClientConfig == nil {
transport.TLSClientConfig = &tls.Config{}
}
transport.TLSClientConfig.InsecureSkipVerify = skip
}
}
func (c Client) SetDefaultTLSConfig(tlsConfig *tls.Config) {
if c.Client.Transport == nil {
c.Client.Transport = &Transport{
base: &http.Transport{},
}
}
if transport, ok := c.Client.Transport.(*Transport); ok {
transport.base.TLSClientConfig = tlsConfig
} else if transport, ok := c.Client.Transport.(*http.Transport); ok {
transport.TLSClientConfig = tlsConfig
}
}
func (c Client) NewRequest(url, method string, opts ...RequestOpt) (*Request, error) {
if c.Client == nil {
return nil, fmt.Errorf("http client is nil")
}
req, err := NewRequestWithContextWithClient(context.Background(), c, url, method, opts...)
return req, err
}
func (c Client) NewRequestContext(ctx context.Context, url, method string, opts ...RequestOpt) (*Request, error) {
if c.Client == nil {
return nil, fmt.Errorf("http client is nil")
}
req, err := NewRequestWithContextWithClient(ctx, c, url, method, opts...)
return req, err
}
func (c Client) NewSimpleRequest(url, method string, opts ...RequestOpt) *Request {
req, _ := c.NewRequest(url, method, opts...)
return req
}
func (c Client) NewSimpleRequestContext(ctx context.Context, url, method string, opts ...RequestOpt) *Request {
req, _ := c.NewRequestContext(ctx, url, method, opts...)
return req
}
type Transport struct {
base *http.Transport
}
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
if t.base == nil {
t.base = &http.Transport{}
}
transport, ok := req.Context().Value("transport").(*http.Transport)
if ok && transport != nil {
return transport.RoundTrip(req)
}
proxy, ok := req.Context().Value("proxy").(string)
if ok && proxy != "" {
tlsConfig, ok := req.Context().Value("tlsConfig").(*tls.Config)
if ok && tlsConfig != nil {
tmpTransport := t.base.Clone()
tmpTransport.TLSClientConfig = tlsConfig
return tmpTransport.RoundTrip(req)
}
}
return t.base.RoundTrip(req)
}

198
curlbench_test.go Normal file
View File

@ -0,0 +1,198 @@
package starnet
import (
"fmt"
"net/http"
"net/http/httptest"
"runtime"
"testing"
)
// BenchmarkGetRequest 测试单个 GET 请求的性能
func BenchmarkGetRequest(b *testing.B) {
// 创建测试服务器
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write([]byte(`OK`))
}))
defer server.Close()
// 重置计时器,排除设置代码的影响
b.ResetTimer()
// 报告内存分配情况
b.ReportAllocs()
// 运行基准测试
for i := 0; i < b.N; i++ {
resp, err := Get(server.URL, WithSkipTLSVerify(true))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != "OK" {
b.Errorf("Expected OK, got %v", body)
}
}
}
// BenchmarkGetRequestWithHeaders 测试带请求头的 GET 请求性能
func BenchmarkGetRequestWithHeaders(b *testing.B) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
// 验证请求头
if req.Header.Get("hello") != "world" {
rw.WriteHeader(http.StatusBadRequest)
return
}
rw.Write([]byte(`OK`))
}))
defer server.Close()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
resp, err := Get(server.URL,
WithSkipTLSVerify(true),
WithHeader("hello", "world"),
WithUserAgent("hello world"))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != "OK" {
b.Errorf("Expected OK, got %v", body)
}
}
}
// BenchmarkPostRequest 测试 POST 请求的性能
func BenchmarkPostRequest(b *testing.B) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
// 读取并返回请求体
body := make([]byte, req.ContentLength)
req.Body.Read(body)
rw.Write(body)
}))
defer server.Close()
testData := "This is a test payload for POST request"
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
resp, err := Post(server.URL,
WithSkipTLSVerify(true),
WithBytes([]byte(testData)),
WithContentType("text/plain"))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != testData {
b.Errorf("Expected %s, got %v", testData, body)
}
}
}
// BenchmarkConcurrentRequests 测试并发请求性能
func BenchmarkConcurrentRequests(b *testing.B) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write([]byte(`OK`))
}))
defer server.Close()
b.ResetTimer()
b.ReportAllocs()
// 运行并发基准测试
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
resp, err := Get(server.URL, WithSkipTLSVerify(true))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != "OK" {
b.Errorf("Expected OK, got %v", body)
}
}
})
}
// BenchmarkMemoryUsage 专门测试内存使用情况
func BenchmarkMemoryUsage(b *testing.B) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write([]byte(`OK`))
}))
defer server.Close()
// 禁用默认的测试时间,只关注内存分配
b.ReportAllocs()
var memStatsStart, memStatsEnd runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&memStatsStart)
for i := 0; i < b.N; i++ {
resp, err := Get(server.URL, WithSkipTLSVerify(true))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().String()
if body != "OK" {
b.Errorf("Expected OK, got %v", body)
}
}
runtime.GC()
runtime.ReadMemStats(&memStatsEnd)
// 计算每次操作的平均内存分配
allocsPerOp := float64(memStatsEnd.Mallocs-memStatsStart.Mallocs) / float64(b.N)
bytesPerOp := float64(memStatsEnd.TotalAlloc-memStatsStart.TotalAlloc) / float64(b.N)
b.ReportMetric(allocsPerOp, "allocs/op")
b.ReportMetric(bytesPerOp, "bytes/op")
}
// BenchmarkDifferentResponseSizes 测试不同响应大小的性能
func BenchmarkDifferentResponseSizes(b *testing.B) {
// 测试不同大小的响应
responseSizes := []int{100, 1024, 10240, 102400} // 100B, 1KB, 10KB, 100KB
for _, size := range responseSizes {
// 生成指定大小的响应数据
responseData := make([]byte, size)
for i := 0; i < size; i++ {
responseData[i] = 'A'
}
b.Run(fmt.Sprintf("Size_%d", size), func(b *testing.B) {
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write(responseData)
}))
defer server.Close()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
resp, err := Get(server.URL, WithSkipTLSVerify(true))
if err != nil {
b.Errorf("Unexpected error: %v", err)
}
body := resp.Body().Bytes()
if len(body) != size {
b.Errorf("Expected size %d, got %d", size, len(body))
}
}
})
}
}