mirror/libowfat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

autofree mode for io_batch

Browse Source
This commit is contained in:
leitner 2024-04-08 12:35:11 +00:00
parent f045b33647
commit 5dbad67352
14 changed files with 194 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Makefile
View File

@ -614,21 +614,30 @@ iob_addfile.o: io/iob_addfile.c iob_internal.h iob.h libowfat/io.h \
iob_addfile_close.o: io/iob_addfile_close.c iob_internal.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/compiler.h libowfat/array.h array.h
iob_adds.o: io/iob_adds.c str.h libowfat/compiler.h iob.h libowfat/io.h \
iob_adds.o: io/iob_adds.c iob_internal.h iob.h libowfat/io.h \
libowfat/uint64.h libowfat/taia.h libowfat/tai.h libowfat/uint32.h \
libowfat/array.h
iob_adds_free.o: io/iob_adds_free.c str.h libowfat/compiler.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/array.h
iob_bytesleft.o: io/iob_bytesleft.c iob.h libowfat/io.h libowfat/uint64.h \
libowfat/taia.h libowfat/tai.h libowfat/uint32.h libowfat/compiler.h \
libowfat/array.h
libowfat/compiler.h libowfat/array.h array.h
iob_adds_free.o: io/iob_adds_free.c iob_internal.h iob.h libowfat/io.h \
libowfat/uint64.h libowfat/taia.h libowfat/tai.h libowfat/uint32.h \
libowfat/compiler.h libowfat/array.h array.h
iob_bytesleft.o: io/iob_bytesleft.c iob_internal.h iob.h libowfat/io.h \
libowfat/uint64.h libowfat/taia.h libowfat/tai.h libowfat/uint32.h \
libowfat/compiler.h libowfat/array.h array.h
iob_free.o: io/iob_free.c iob_internal.h iob.h libowfat/io.h \
libowfat/uint64.h libowfat/taia.h libowfat/tai.h libowfat/uint32.h \
libowfat/compiler.h libowfat/array.h array.h
iob_init.o: io/iob_init.c byte.h libowfat/compiler.h iob_internal.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/array.h array.h
iob_init_autofree.o: io/iob_init_autofree.c iob_internal.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/compiler.h libowfat/array.h array.h
iob_new.o: io/iob_new.c byte.h libowfat/compiler.h iob_internal.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/array.h array.h
iob_new_autofree.o: io/iob_new_autofree.c iob_internal.h iob.h \
libowfat/io.h libowfat/uint64.h libowfat/taia.h libowfat/tai.h \
libowfat/uint32.h libowfat/compiler.h libowfat/array.h array.h
iob_prefetch.o: io/iob_prefetch.c iob_internal.h iob.h libowfat/io.h \
libowfat/uint64.h libowfat/taia.h libowfat/tai.h libowfat/uint32.h \
libowfat/compiler.h libowfat/array.h array.h
@ -883,7 +892,8 @@ socket_tcp4.o: socket/socket_tcp4.c socket.h libowfat/compiler.h \
socket_tcp4b.o: socket/socket_tcp4b.c windoze.h socket.h \
libowfat/compiler.h libowfat/uint16.h libowfat/uint32.h ndelay.h
socket_tcp6.o: socket/socket_tcp6.c haveip6.h socket.h \
libowfat/compiler.h libowfat/uint16.h libowfat/uint32.h ndelay.h
libowfat/compiler.h libowfat/uint16.h libowfat/uint32.h ndelay.h \
windoze.h
socket_tcp6b.o: socket/socket_tcp6b.c windoze.h haveip6.h socket.h \
libowfat/compiler.h libowfat/uint16.h libowfat/uint32.h ndelay.h
socket_tryreservein.o: socket/socket_tryreservein.c windoze.h socket.h \
@ -1080,7 +1090,7 @@ DNS_OBJS=dns_dfd.o dns_domain.o dns_dtda.o dns_ip.o dns_ip6.o dns_ipq.o dns_ipq6
CASE_OBJS=case_diffb.o case_diffs.o case_lowerb.o case_lowers.o case_starts.o
MULT_OBJS=imult16.o imult32.o imult64.o range_arrayinbuf.o range_str2inbuf.o range_str4inbuf.o range_strinbuf.o umult16.o umult32.o umult64.o
ARRAY_OBJS=array_allocate.o array_bytes.o array_cat.o array_cat0.o array_catb.o array_cate.o array_cats.o array_cats0.o array_equal.o array_fail.o array_get.o array_length.o array_reset.o array_start.o array_trunc.o array_truncate.o iarray_allocate.o iarray_free.o iarray_get.o iarray_init.o iarray_length.o
IO_OBJS=io_appendfile.o io_block.o io_canread.o io_canwrite.o io_check.o io_close.o io_closeonexec.o io_createfile.o io_debugstring.o io_dontwantread.o io_dontwantwrite.o io_eagain.o io_eagain_read.o io_eagain_write.o io_fd.o io_finishandshutdown.o io_getcookie.o io_mmapwritefile.o io_nonblock.o io_passfd.o io_pipe.o io_readfile.o io_readwritefile.o io_receivefd.o io_sendfile.o io_setcookie.o io_sigpipe.o io_socketpair.o io_timedout.o io_timeout.o io_timeouted.o io_tryread.o io_tryreadtimeout.o io_trywrite.o io_trywritetimeout.o io_wait.o io_waitread.o io_waituntil.o io_waituntil2.o io_waitwrite.o io_wantread.o io_wantwrite.o iob_addbuf.o iob_addbuf_free.o iob_addbuf_internal.o iob_addbuf_munmap.o iob_addfile.o iob_addfile_close.o iob_adds.o iob_adds_free.o iob_bytesleft.o iob_free.o iob_new.o iob_prefetch.o iob_reset.o iob_send.o iob_write.o
IO_OBJS=io_appendfile.o io_block.o io_canread.o io_canwrite.o io_check.o io_close.o io_closeonexec.o io_createfile.o io_debugstring.o io_dontwantread.o io_dontwantwrite.o io_eagain.o io_eagain_read.o io_eagain_write.o io_fd.o io_finishandshutdown.o io_getcookie.o io_mmapwritefile.o io_nonblock.o io_passfd.o io_pipe.o io_readfile.o io_readwritefile.o io_receivefd.o io_sendfile.o io_setcookie.o io_sigpipe.o io_socketpair.o io_timedout.o io_timeout.o io_timeouted.o io_tryread.o io_tryreadtimeout.o io_trywrite.o io_trywritetimeout.o io_wait.o io_waitread.o io_waituntil.o io_waituntil2.o io_waitwrite.o io_wantread.o io_wantwrite.o iob_addbuf.o iob_addbuf_free.o iob_addbuf_internal.o iob_addbuf_munmap.o iob_addfile.o iob_addfile_close.o iob_adds.o iob_adds_free.o iob_bytesleft.o iob_free.o iob_new.o iob_prefetch.o iob_reset.o iob_send.o iob_write.o iom_abort.o iom_add.o iom_init.o iom_wait.o
TEXTCODE_OBJS=base64.o base64url.o fmt_base64.o fmt_base64url.o fmt_cescape.o fmt_foldwhitespace.o fmt_hexdump.o fmt_html.o fmt_html_tagarg.o fmt_jsonescape.o fmt_ldapescape.o fmt_ldapescape2.o fmt_quotedprintable.o fmt_to_array.o fmt_to_sa.o fmt_tofrom_array.o fmt_urlencoded.o fmt_uuencoded.o fmt_xml.o fmt_yenc.o scan_base64.o scan_base64url.o scan_cescape.o scan_hexdump.o scan_html.o scan_jsonescape.o scan_ldapescape.o scan_quotedprintable.o scan_to_array.o scan_to_sa.o scan_tofrom_array.o scan_urlencoded.o scan_uuencoded.o scan_yenc.o
CDB_OBJS=cdb.o cdb_hash.o cdb_make.o cdb_traverse.o
CRITBIT_OBJS=critbit.o
@ -1180,7 +1190,7 @@ uninstall:
rm -f $(LIBDIR)/libowfat.a
VERSION=libowfat-0.34
CURNAME=libowfat-0.32
CURNAME=libowfat-0.34
tar: clean rename compile_commands.json
rm -f dep libdep

6
io/iob_adds.c
View File

@ -1,6 +1,6 @@
#include "str.h"
#include "iob.h"
#include <string.h>
#include "iob_internal.h"
int iob_adds(io_batch* b,const char* s) {
return iob_addbuf(b,s,str_len(s));
return iob_addbuf(b,s,strlen(s));
}

6
io/iob_adds_free.c
View File

@ -1,6 +1,6 @@
#include "str.h"
#include "iob.h"
#include <string.h>
#include "iob_internal.h"
int iob_adds_free(io_batch* b,const char* s) {
return iob_addbuf_free(b,s,str_len(s));
return iob_addbuf_free(b,s,strlen(s));
}

2
io/iob_bytesleft.c
View File

@ -1,4 +1,4 @@
#include "iob.h"
#include "iob_internal.h"
uint64 iob_bytesleft(const io_batch* b) {
return b->bytesleft;

13
io/iob_init.c Normal file
View File

@ -0,0 +1,13 @@
#include <stdlib.h>
#include "byte.h"
#include "iob_internal.h"
int iob_init(io_batch* b,size_t hint_entries) {
byte_zero(b,sizeof(io_batch));
if (hint_entries) {
if (!array_allocate(&b->b,sizeof(iob_entry),hint_entries))
return -1;
array_trunc(&b->b);
}
return 0;
}

7
io/iob_init_autofree.c Normal file
View File

@ -0,0 +1,7 @@
#include "iob_internal.h"
int iob_init_autofree(io_batch* b,size_t hint_entries) {
int r=iob_init(b,hint_entries);
b->autofree=1;
return r;
}

2
io/iob_new.3
View File

@ -4,7 +4,7 @@ iob_new \- create new I/O batch
.SH SYNTAX
.B #include <libowfat/iob.h>
io_batch* \fBiob_new\fP(int hint_entries);
io_batch* \fBiob_new\fP(size_t hint_entries);
.SH DESCRIPTION
iob_new creates a new I/O batch with enough space allocated for
\fIhint_entries\fR entries (buffers or files). This is purely a

2
io/iob_new.c
View File

@ -2,7 +2,7 @@
#include "byte.h"
#include "iob_internal.h"
io_batch* iob_new(int hint_entries) {
io_batch* iob_new(size_t hint_entries) {
io_batch* b=(io_batch*)malloc(sizeof(io_batch));
if (!b) return 0;
byte_zero(b,sizeof(io_batch));

8
io/iob_new_autofree.c Normal file
View File

@ -0,0 +1,8 @@
#include "iob_internal.h"
io_batch* iob_new_autofree(size_t hint_entries) {
io_batch* b=iob_new(hint_entries);
if (b)
b->autofree=1;
return b;
}

78
io/iob_send.c
View File

@ -7,6 +7,14 @@
#include "iob_internal.h"
#include <stdio.h>
/*
__ ___ _
\ \ / (_)_ __ __| | _____ _____
\ \ /\ / /| | '_ \ / _` |/ _ \ \ /\ / / __|
\ V V / | | | | | (_| | (_) \ V V /\__ \
\_/\_/ |_|_| |_|\__,_|\___/ \_/\_/ |___/
*/
int64 iob_send(int64 s,io_batch* b) {
/* Windows has a sendfile called TransmitFile, which can send one
* header and one trailer buffer. */
@ -108,6 +116,14 @@ int64 iob_send(int64 s,io_batch* b) {
#else
/*
_ _ _
| | | |_ __ (_)_ __
| | | | '_ \| \ \/ /
| |_| | | | | |> <
\___/|_| |_|_/_/\_\
*/
#include "havebsdsf.h"
#include <sys/types.h>
#include <sys/socket.h>
@ -145,7 +161,7 @@ int64 iob_send(int64 s,io_batch* b) {
if (b->bytesleft==0) return 0;
E=iarray_get(&io_fds,s);
if (!E) { errno=EBADF; return -3; }
// if (!E) { errno=EBADF; return -3; }
last=(iob_entry*)(((char*)array_start(&b->b))+array_bytes(&b->b));
v=alloca(b->bufs*sizeof(struct iovec));
total=0;
@ -167,6 +183,14 @@ int64 iob_send(int64 s,io_batch* b) {
}
headers=i;
#ifdef HAVE_BSDSENDFILE
/*
____ ____ ____
| __ ) ___|| _ \
| _ \___ \| | | |
| |_) |__) | |_| |
|____/____/|____/
*/
if (e+i<last && e[i].type==FROMFILE) {
#ifdef DEBUGONLINUX
@ -212,6 +236,14 @@ eagain:
}
}
#else
/*
_ _
| | (_)_ __ _ ___ __
| | | | '_ \| | | \ \/ /
| |___| | | | | |_| |> <
|_____|_|_| |_|\__,_/_/\_\
*/
/* Linux has two ways to coalesce sent data; either setsockopt
* TCP_CORK or sendto/sendmsg with MSG_MORE. MSG_MORE saves syscalls
* in one scenario: when there is n buffers and then possibly one
@ -245,7 +277,7 @@ eagain:
if (!nozerocopy && sum>=8*1024) {
/* MSG_ZEROCOPY has page table management overhead,
* it only pays off after 8k or so */
if (E->zerocopy==0) {
if (E && E->zerocopy==0) {
if (setsockopt(s, SOL_SOCKET, SO_ZEROCOPY, (int[]){ 1 },sizeof(int)) == 0) {
E->zerocopy=1;
ZEROCOPY=MSG_ZEROCOPY;
@ -273,9 +305,18 @@ eagain:
n = headers-skip; if (n > 50) n=50;
for (i=0; i<n; ++i) l += v[skip+i].iov_len;
// printf("writing %d records from offset %d, %d bytes\n", skip, n, l);
msg.msg_iov=v + skip;
msg.msg_iovlen=n;
sent=sendmsg(s,&msg,MSG_MORE|ZEROCOPY);
if (E && E->notsock)
notsockwritev:
sent=writev(s,v+skip,n);
else {
msg.msg_iov=v + skip;
msg.msg_iovlen=n;
sent=sendmsg(s,&msg,MSG_MORE|ZEROCOPY);
if (sent==-1 && errno==ENOTSOCK) {
if (E) E->notsock=1;
goto notsockwritev;
}
}
if (sent > 0) totalsent += sent;
if (sent == l) continue; // we sent as much as we wanted, go for next batch
if (sent >= 0) // we wrote something but not the whole batch
@ -294,8 +335,20 @@ eagain:
// if we get here, we wrote it all or we got an EAGAIN after
// writing something. Treat as regular partial write.
sent = totalsent;
} else
sent=sendmsg(s,&msg,MSG_MORE|ZEROCOPY);
} else {
if (E && E->notsock)
notsockwritev2:
sent=writev(s,v,headers);
else {
msg.msg_iov=v;
msg.msg_iovlen=headers;
sent=sendmsg(s,&msg,MSG_MORE|ZEROCOPY);
if (sent==-1 && errno==ENOTSOCK) {
if (E) E->notsock=1;
goto notsockwritev2;
}
}
}
}
}
if (sent==-1) {
@ -345,6 +398,11 @@ eagain:
for (i=0; e+i<last; ++i) {
if (e[i].n<=rest) {
rest-=e[i].n;
if (b->autofree) {
if (e[i].cleanup)
e[i].cleanup(e+i);
e[i].cleanup=0;
}
++b->next;
if (!rest) break;
} else {
@ -353,8 +411,10 @@ eagain:
goto abort;
}
}
io_eagain_write(s);
return total;
if (b->bytesleft==0) {
io_eagain_write(s);
return total;
}
} else break;
}
abort:

9
io/iob_write.c
View File

@ -11,7 +11,7 @@ int64 iob_write2(int64 s,io_batch* b,io_write_callback cb,io_sendfile_callback s
total=0;
if (!(e=array_get(&b->b,sizeof(iob_entry),b->next)))
return -3; /* can't happen error */
for (i=0; e+i<last; ++i) {
for (i=b->next; e+i<last; ++i) {
int thatsit;
if (!e[i].n) continue;
if (e[i].type==FROMFILE)
@ -27,6 +27,13 @@ int64 iob_write2(int64 s,io_batch* b,io_write_callback cb,io_sendfile_callback s
total+=sent;
b->bytesleft-=sent;
if (thatsit) break;
// we have sent one full entry
if (b->autofree) {
if (e[i].cleanup)
e[i].cleanup(e+i);
e[i].cleanup=0;
}
++b->next;
}
if (total == b->bytesleft)
iob_reset(b);

1
io_internal.h
View File

@ -50,6 +50,7 @@ typedef struct {
unsigned int closed:1; /* io_close called, but close deferred because of outstanding events */
unsigned int zerocopy:1; /* linux: setsockopt SO_ZEROCOPY done */
unsigned int goterror:1; /* got POLLERR|POLLHUP */
unsigned int notsock:1; /* linux: got ENOTSOCKET from sendmsg, use writev */
#ifdef __MINGW32__
unsigned int readqueued:2;
unsigned int writequeued:2;

51
iob.h
View File

@ -21,34 +21,83 @@
extern "C" {
#endif
/* you should not have to touch any of the internals */
typedef struct io_batch {
/* array of iob_entry, defined in iob_internal.h */
array b;
/* bytes left in the batch (starts as sum of lengths of all iob_entries */
uint64 bytesleft;
/* next is the index of the first unsent batch in the array */
long next,bufs,files;
/* Ask iob_send to free entries from the array as we go.
* Usually when iob_send returns 0 or error, you call iob_reset.
* You still have to call iob_reset even with this flag set, as
* autofree operates on the buffers in the array but does not free the
* array itself */
int autofree;
} io_batch;
io_batch* iob_new(int hint_entries);
/* Initialize an io_batch. Return 0 on success, -1 on malloc failure for
* embedded array */
int iob_init(io_batch* b,size_t hint_entries);
/* initialize an io_batch that auto-frees entries as soon as
* iob_send/iob_write have written them. Return 0 on success, -1 on
* malloc failure for embedded array */
int iob_init_autofree(io_batch* b,size_t hint_entries);
/* malloc and initialize an io_batch */
io_batch* iob_new(size_t hint_entries);
/* malloc and initialize an io_batch that auto-frees entries as soon as
* iob_send/iob_write have written them */
io_batch* iob_new_autofree(size_t hint_entries);
/* queue buffer in io_batch */
att_readn(2,3)
int iob_addbuf(io_batch* b,const void* buf,uint64 n);
/* queue buffer in io_batch, and give ownership to batch. */
/* the io_batch functions will take care of freeing it. */
att_readn(2,3)
int iob_addbuf_free(io_batch* b,const void* buf,uint64 n);
/* queue mmapped memory reagion in io_batch, and give ownership to batch. */
/* the io_batch functions will take care of unmapping it. */
att_readn(2,3)
int iob_addbuf_munmap(io_batch* b,const void* buf,uint64 n);
/* queue asciiz string in io_batch. */
att_read(2)
int iob_adds(io_batch* b,const char* s);
/* queue asciiz string in io_batch, and give ownership to batch. */
/* the io_batch functions will take care of freeing it. */
att_read(2)
int iob_adds_free(io_batch* b,const char* s);
/* queue file contents in io_batch. */
int iob_addfile(io_batch* b,int64 fd,uint64 off,uint64 n);
/* queue file contents in io_batch, and give ownership to batch */
/* the io_batch functions will take care of closing it. */
int iob_addfile_close(io_batch* b,int64 fd,uint64 off,uint64 n);
/* send (rest of) io_batch over socket s */
/* return number of bytes written.
* return 0 means whole batch written successfully.
* return -1 means couldn't write more on non-blocking socket, try again later
* return -3 means error (errno set) */
int64 iob_send(int64 s,io_batch* b);
/* same as iob_send but don't use callback instead of write and
* sendfile. Use this to write via OpenSSL or so */
int64 iob_write(int64 s,io_batch* b,io_write_callback cb);
/* same as iob_write if the TLS library also has a sendfile function */
int64 iob_write2(int64 s,io_batch* b,io_write_callback cb,io_sendfile_callback sfcb);
/* iob_reset closes files and frees buffer in the io_batch, and the
* array with the buffer, but not the batch itself (in case it's a local
* variable and not malloced) */
void iob_reset(io_batch* b);
/* iob_free does iob_reset but also frees the batch itself */
void iob_free(io_batch* b);
void iob_prefetch(io_batch* b,uint64 bytes);
uint64 iob_bytesleft(const io_batch* b);

14
test/iob.c
View File

@ -1,7 +1,7 @@
#include <assert.h>
#include "iob.h"
#include "buffer.h"
#include "libowfat/iob.h"
#include <unistd.h>
#include <string.h>
static int64 write_cb(int64 fd,const void* buf,uint64 len) {
if (len>2) len=2;
@ -11,14 +11,16 @@ static int64 write_cb(int64 fd,const void* buf,uint64 len) {
int main() {
int64 fd;
io_batch* b;
int64 r;
char* fnord=strdup(" fnord\n");
assert(fnord);
assert(io_readfile(&fd,"GNUmakefile"));
assert(b=iob_new(10));
assert((b=iob_new_autofree(10)));
assert(iob_addbuf(b,"Huhu",4));
assert(iob_addbuf(b," fnord\n",7));
assert(iob_addbuf_free(b,fnord,7));
assert(iob_addfile(b,fd,10,10));
iob_send(1,b);
#if 1
#if 0
int64 r;
do {
r=iob_write(1,b,write_cb);
} while (r>0);