mirror/opentracker
1
0
Fork 0
mirror of git://erdgeist.org/opentracker synced 2025-02-27 19:41:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support for dynamic accesslists

Browse Source
This commit is contained in:
Dirk Engling 2022-11-24 04:20:06 +01:00
parent 110868ec4e
commit be825f5759
8 changed files with 297 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -24,6 +24,7 @@ STRIP?=strip
#FEATURES+=-DWANT_ACCESSLIST_BLACK
#FEATURES+=-DWANT_ACCESSLIST_WHITE
#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
#FEATURES+=-DWANT_SYNC_LIVE
#FEATURES+=-DWANT_IP_FROM_QUERY_STRING

12
opentracker.c
View File

@ -135,8 +135,8 @@ static void help( char *name ) {
}
#undef HELPLINE
static size_t header_complete( char * request, ssize_t byte_count ) {
int i = 0, state = 0;
static ssize_t header_complete( char * request, ssize_t byte_count ) {
ssize_t i = 0, state = 0;
for( i=1; i < byte_count; i+=2 )
if( request[i] <= 13 ) {
@ -184,7 +184,7 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) {
ws->request_size = byte_count;
http_handle_request( sock, ws );
} else
array_catb( &cookie->request, ws->inbuf, byte_count );
array_catb( &cookie->request, ws->inbuf, (size_t)byte_count );
return;
}
@ -463,6 +463,12 @@ int parse_configfile( char * config_filename ) {
} else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) {
set_config_option( &g_accesslist_filename, p+17 );
#endif
#ifdef WANT_DYNAMIC_ACCESSLIST
} else if(!byte_diff(p, 15, "access.fifo_add" ) && isspace(p[15])) {
set_config_option( &g_accesslist_pipe_add, p+16 );
} else if(!byte_diff(p, 18, "access.fifo_delete" ) && isspace(p[18])) {
set_config_option( &g_accesslist_pipe_delete, p+19 );
#endif
#ifdef WANT_RESTRICT_STATS
} else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) {
if( !scan_ip6( p+13, tmpip )) goto parse_error;

37
opentracker.conf.sample
View File

@ -44,6 +44,43 @@
# listing, so choose one of those options at compile time. File format
# is straight forward: "<hex info hash>\n<hex info hash>\n..."
#
# IIa) You can enable dynamic changesets to accesslists by enabling
# WANT_DYNAMIC_ACCESSLIST.
#
# The suggested way to work with dynamic changeset lists is to keep a
# main accesslist file that is loaded when opentracker (re)starts and
# reloaded infrequently (hourly or daily).
#
# All changes to the accesslist (e.g. from a web frontend) should be
# both appended to or removed from that file and sent to opentracker. By
# keeping dynamic changeset lists, you can avoid reloading huge
# accesslists whenever just a single entry is added or removed.
#
# Any info_hash (format see above) written to the fifo_add file will be
# kept on a dynamic add-changeset, removed from the dynamic
# delete-changeset and treated as if it was in the main accesslist file.
# The semantic of the respective dynamic changeset depends on whether
# WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled.
#
# access.fifo_add /var/run/opentracker/adder.fifo
#
# Any info_hash (format see above) written to the fifo_delete file will
# be kept on a dynamic delete-changeset, removed from the dynamic
# add-changeset and treated as if it was not in the main accesslist
# file.
#
# access.fifo_delete /var/run/opentracker/deleter.fifo
#
# If you reload the accesslist by sending SIGHUP to the tracker process,
# the dynamic lists are flushed, as opentracker assumes thoses lists are
# merged into the main accesslist.
#
# NOTE: While you can have multiple writers sending lines to the fifos,
# any writes larger than PIPE_BUF (see your limits.h, minimally 512
# bytes but usually 4096) may be interleaved with data sent by other
# writers. This can lead to unparsable lines of info_hashes.
#
# IIb)
# If you do not want to grant anyone access to your stats, enable the
# WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
# allowed to fetch stats here.

271
ot_accesslist.c
View File

@ -10,6 +10,11 @@
#include <stdio.h>
#include <signal.h>
#include <unistd.h>
#ifdef WANT_DYNAMIC_ACCESSLIST
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#endif
/* Libowfat */
#include "byte.h"
@ -24,23 +29,80 @@
/* GLOBAL VARIABLES */
#ifdef WANT_ACCESSLIST
char *g_accesslist_filename;
char *g_accesslist_filename = NULL;
#ifdef WANT_DYNAMIC_ACCESSLIST
char *g_accesslist_pipe_add = NULL;
char *g_accesslist_pipe_delete = NULL;
#endif
static pthread_mutex_t g_accesslist_mutex;
typedef struct {
ot_hash *list;
size_t size;
} ot_accesslist;
ot_accesslist * g_accesslist = NULL;
ot_accesslist * g_accesslist_old = NULL;
/* Accesslists are lock free linked lists. We can not make them locking, because every announce
would try to acquire the mutex, making it the most contested mutex in the whole of opentracker,
basically creating a central performance choke point.
The idea is that updating the list heads happens under the g_accesslist_mutex guard and is
done atomically, while consumers might potentially still hold pointers deeper inside the list.
Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer
that is guaranteed to live for at least five minutes. This should be many orders of magnitudes
more than how long it will be needed by the bsearch done on the list. */
struct ot_accesslist;
typedef struct ot_accesslist ot_accesslist;
struct ot_accesslist {
ot_hash *list;
size_t size;
ot_time base;
ot_accesslist *next;
};
static ot_accesslist * g_accesslist = NULL;
#ifdef WANT_DYNAMIC_ACCESSLIST
static ot_accesslist * g_accesslist_add = NULL;
static ot_accesslist * g_accesslist_delete = NULL;
#endif
/* Helpers to work on access lists */
static int vector_compare_hash(const void *hash1, const void *hash2 ) {
return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE );
}
static ot_accesslist * accesslist_free(ot_accesslist *accesslist) {
while (accesslist) {
ot_accesslist * this_accesslist = accesslist;
accesslist = this_accesslist->next;
free(this_accesslist->list);
free(this_accesslist);
}
return NULL;
}
static ot_accesslist * accesslist_make(ot_accesslist *next, size_t size) {
ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist));
if (accesslist_new) {
accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL;
accesslist_new->size = size;
accesslist_new->base = g_now_minutes;
accesslist_new->next = next;
if (size && !accesslist_new->list) {
free(accesslist_new);
accesslist_new = NULL;
}
}
return accesslist_new;
}
/* This must be called with g_accesslist_mutex held.
This will never delete head, because that might still be in use. */
static void accesslist_clean(ot_accesslist *accesslist) {
while (accesslist && accesslist->next) {
if (accesslist->next->base + 5 < g_now_minutes)
accesslist->next = accesslist_free(accesslist->next);
accesslist = accesslist->next;
}
}
/* Read initial access list */
static void accesslist_readfile( void ) {
ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist));
ot_accesslist * accesslist_new;
ot_hash *info_hash;
const char *map, *map_end, *read_offs;
size_t maplen;
@ -54,14 +116,13 @@ static void accesslist_readfile( void ) {
/* You need at least 41 bytes to pass an info_hash, make enough room
for the maximum amount of them */
accesslist_new->size = 0;
info_hash = accesslist_new->list = malloc( ( maplen / 41 ) * 20 );
if( !accesslist_new->list ) {
accesslist_new = accesslist_make(g_accesslist, maplen / 41);
if( !accesslist_new ) {
fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 );
mmap_unmap( map, maplen);
free(accesslist_new);
return;
}
info_hash = accesslist_new->list;
/* No use to scan if there's not enough room for another full info_hash */
map_end = map + maplen - 40;
@ -71,18 +132,18 @@ static void accesslist_readfile( void ) {
while( read_offs <= map_end ) {
int i;
for( i=0; i<(int)sizeof(ot_hash); ++i ) {
int eger1 = scan_fromhex( read_offs[ 2*i ] );
int eger2 = scan_fromhex( read_offs[ 1 + 2*i ] );
int eger1 = scan_fromhex( (unsigned char)read_offs[ 2*i ] );
int eger2 = scan_fromhex( (unsigned char)read_offs[ 1 + 2*i ] );
if( eger1 < 0 || eger2 < 0 )
break;
(*info_hash)[i] = eger1 * 16 + eger2;
(*info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
}
if( i == sizeof(ot_hash) ) {
read_offs += 40;
/* Append accesslist to accesslist vector */
if( read_offs == map_end || scan_fromhex( *read_offs ) < 0 )
if( read_offs == map_end || scan_fromhex( (unsigned char)*read_offs ) < 0 )
++info_hash;
}
@ -100,14 +161,19 @@ static void accesslist_readfile( void ) {
/* Now exchange the accesslist vector in the least race condition prone way */
pthread_mutex_lock(&g_accesslist_mutex);
accesslist_new->next = g_accesslist;
g_accesslist = accesslist_new; /* Only now set a new list */
if (g_accesslist_old) {
free(g_accesslist_old->list);
free(g_accesslist_old);
}
#ifdef WANT_DYNAMIC_ACCESSLIST
/* If we have dynamic accesslists, reloading a new one will always void the add/delete lists.
Insert empty ones at the list head */
if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL)
g_accesslist_add = accesslist_new;
if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL)
g_accesslist_delete = accesslist_new;
#endif
g_accesslist_old = g_accesslist; /* Keep a copy for later free */
g_accesslist = accesslist_new; /* Only now set a new list */
accesslist_clean(g_accesslist);
pthread_mutex_unlock(&g_accesslist_mutex);
}
@ -115,11 +181,25 @@ static void accesslist_readfile( void ) {
int accesslist_hashisvalid( ot_hash hash ) {
/* Get working copy of current access list */
ot_accesslist * accesslist = g_accesslist;
#ifdef WANT_DYNAMIC_ACCESSLIST
ot_accesslist * accesslist_add, * accesslist_delete;
#endif
void * exactmatch = NULL;
if (accesslist)
exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
#ifdef WANT_DYNAMIC_ACCESSLIST
/* If we had no match on the main list, scan the list of dynamically added hashes */
accesslist_add = g_accesslist_add;
if ((exactmatch == NULL) && accesslist_add)
exactmatch = bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
/* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */
accesslist_delete = g_accesslist_delete;
if ((exactmatch != NULL) && accesslist_delete && bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ))
exactmatch = NULL;
#endif
#ifdef WANT_ACCESSLIST_BLACK
return exactmatch == NULL;
@ -138,6 +218,8 @@ static void * accesslist_worker( void * args ) {
(void)args;
while( 1 ) {
if (!g_opentracker_running)
return NULL;
/* Initial attempt to read accesslist */
accesslist_readfile( );
@ -148,27 +230,144 @@ static void * accesslist_worker( void * args ) {
return NULL;
}
#ifdef WANT_DYNAMIC_ACCESSLIST
static pthread_t thread_adder_id, thread_deleter_id;
static void * accesslist_adddel_worker(char * fifoname, ot_accesslist ** adding_to, ot_accesslist ** removing_from) {
struct stat st;
if (!stat(fifoname, &st)) {
if (!S_ISFIFO(st.st_mode)) {
fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname);
return NULL;
}
} else {
int error = mkfifo(fifoname, 0755);
if (error && error != EEXIST) {
fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno));
return NULL;
}
}
while (g_opentracker_running) {
FILE * fifo = fopen(fifoname, "r");
char *line = NULL;
size_t linecap = 0;
ssize_t linelen;
if (!fifo) {
fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno));
return NULL;
}
while ((linelen = getline(&line, &linecap, fifo)) > 0) {
ot_hash info_hash;
int i;
printf("Got line %*s", (int)linelen, line);
/* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*"
If there's not enough characters for an info_hash in the line, skip it. */
if (linelen < 41)
continue;
for( i=0; i<(int)sizeof(ot_hash); ++i ) {
int eger1 = scan_fromhex( (unsigned char)line[ 2*i ] );
int eger2 = scan_fromhex( (unsigned char)line[ 1 + 2*i ] );
if( eger1 < 0 || eger2 < 0 )
break;
((uint8_t*)info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
}
printf("parsed info_hash %20s\n", info_hash);
if( i != sizeof(ot_hash) )
continue;
/* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the
other worker threads from doing the same */
pthread_mutex_lock(&g_accesslist_mutex);
/* If the info hash is in the removing_from list, create a new head without that entry */
if (*removing_from && (*removing_from)->list) {
ot_hash * exactmatch = bsearch( info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
if (exactmatch) {
ptrdiff_t off = exactmatch - (*removing_from)->list;
ot_accesslist * accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1);
if (accesslist_new) {
memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off);
memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, (*removing_from)->size - off - 1);
*removing_from = accesslist_new;
}
}
}
accesslist_clean(*removing_from);
/* Simple case: there's no adding_to list yet, create one with one member */
if (!*adding_to) {
*adding_to = accesslist_make(NULL, 1);
if (*adding_to)
memcpy((*adding_to)->list, info_hash, sizeof(ot_hash));
} else {
int exactmatch = 0;
ot_hash * insert_point = binary_search( info_hash, (*adding_to)->list, (*adding_to)->size, OT_HASH_COMPARE_SIZE, sizeof(ot_hash), &exactmatch );
/* Only if the info hash is not in the adding_to list, create a new head with that entry */
if (!exactmatch) {
ot_accesslist * accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1);
ptrdiff_t off = insert_point - (*adding_to)->list;
if (accesslist_new) {
memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off);
memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash));
memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, (*adding_to)->size - off);
*adding_to = accesslist_new;
}
}
}
accesslist_clean(*adding_to);
pthread_mutex_unlock(&g_accesslist_mutex);
}
fclose(fifo);
}
return NULL;
}
static void * accesslist_adder_worker( void * args ) {
(void)args;
return accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete);
}
static void * accesslist_deleter_worker( void * args ) {
(void)args;
return accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add);
}
#endif
static pthread_t thread_id;
void accesslist_init( ) {
pthread_mutex_init(&g_accesslist_mutex, NULL);
pthread_create( &thread_id, NULL, accesslist_worker, NULL );
#ifdef WANT_DYNAMIC_ACCESSLIST
if (g_accesslist_pipe_add)
pthread_create( &thread_adder_id, NULL, accesslist_adder_worker, NULL );
if (g_accesslist_pipe_delete)
pthread_create( &thread_deleter_id, NULL, accesslist_deleter_worker, NULL );
#endif
}
void accesslist_deinit( void ) {
/* Wake up sleeping worker */
pthread_kill(thread_id, SIGHUP);
pthread_mutex_lock(&g_accesslist_mutex);
g_accesslist = accesslist_free(g_accesslist);
#ifdef WANT_DYNAMIC_ACCESSLIST
g_accesslist_add = accesslist_free(g_accesslist_add);
g_accesslist_delete = accesslist_free(g_accesslist_delete);
#endif
pthread_mutex_unlock(&g_accesslist_mutex);
pthread_cancel( thread_id );
pthread_mutex_destroy(&g_accesslist_mutex);
if (g_accesslist_old) {
free(g_accesslist_old->list);
free(g_accesslist_old);
g_accesslist_old = 0;
}
if (g_accesslist) {
free(g_accesslist->list);
free(g_accesslist);
g_accesslist = 0;
}
}
#endif

8
ot_accesslist.h
View File

@ -12,11 +12,15 @@
#if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE )
#define WANT_ACCESSLIST
void accesslist_init( );
void accesslist_deinit( );
void accesslist_init( void );
void accesslist_deinit( void );
int accesslist_hashisvalid( ot_hash hash );
extern char *g_accesslist_filename;
#ifdef WANT_DYNAMIC_ACCESSLIST
extern char *g_accesslist_pipe_add;
extern char *g_accesslist_pipe_delete;
#endif
#else
#define accesslist_init( accesslist_filename )

6
ot_mutex.h
View File

@ -8,8 +8,8 @@
#include <sys/uio.h>
void mutex_init( );
void mutex_deinit( );
void mutex_init( void );
void mutex_deinit( void );
ot_vector *mutex_bucket_lock( int bucket );
ot_vector *mutex_bucket_lock_by_hash( ot_hash hash );
@ -17,7 +17,7 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash );
void mutex_bucket_unlock( int bucket, int delta_torrentcount );
void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount );
size_t mutex_get_torrent_count();
size_t mutex_get_torrent_count(void);
typedef enum {
TASK_STATS_CONNS = 0x0001,

6
ot_stats.h
View File

@ -40,11 +40,11 @@ enum {
void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data );
void stats_deliver( int64 sock, int tasktype );
void stats_cleanup();
void stats_cleanup( void );
size_t return_stats_for_tracker( char *reply, int mode, int format );
size_t stats_return_tracker_version( char *reply );
void stats_init( );
void stats_deinit( );
void stats_init( void );
void stats_deinit( void );
extern const char *g_version_rijndael_c;
extern const char *g_version_livesync_c;

5
trackerlogic.h
View File

@ -133,7 +133,6 @@ struct ot_workstruct {
char *peer_id;
/* HTTP specific, non static */
int keep_alive;
char *request;
ssize_t request_size;
ssize_t header_size;
@ -143,6 +142,8 @@ struct ot_workstruct {
/* Entropy state for rand48 function so that threads don't need to acquire mutexes for
global random() or arc4random() state, which causes heavy load on linuxes */
uint16_t rand48_state[3];
int keep_alive;
};
/*
@ -163,7 +164,7 @@ struct ot_workstruct {
#error Live logging networks disabled at the moment.
#endif
void trackerlogic_init( );
void trackerlogic_init( void );
void trackerlogic_deinit( void );
void exerr( char * message );