Add proper parsing for the gzip content encoding

Makefile

@@ -24,7 +24,7 @@ STRIP?=strip
 #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
 
 #FEATURES+=-DWANT_SYNC_LIVE
-#FEATURES+=-DWANT_IP_FROM_QUERY_STRING
+FEATURES+=-DWANT_IP_FROM_QUERY_STRING
 #FEATURES+=-DWANT_COMPRESSION_GZIP
 #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
 #FEATURES+=-DWANT_LOG_NETWORKS
@@ -48,7 +48,7 @@ FEATURES+=-DWANT_FULLSCRAPE
 OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
 OPTS_production=-O3
 
-CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra #-ansi -pedantic
+CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra -Wincompatible-pointer-types #-ansi -pedantic
 LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz
 #LDFLAGS+=-lbsd
 

ot_fullscrape.c

@@ -262,33 +262,36 @@ fprintf(stderr, "GZIP path\n");
   if( deflate( &strm, Z_FINISH ) < Z_OK )
     fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
 
-  if( !strm.avail_out ) {
+  iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
+  if (mutex_workqueue_pushchunked(taskid, &iovector) ) {
+      free(iovector.iov_base);
+      return mutex_bucket_unlock( bucket, 0 );
+  }
+
+  {
     unsigned int pending;
     int bits;
     deflatePending( &strm, &pending, &bits);
     pending += ( bits ? 1 : 0 );
 
-    iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
-    if (mutex_workqueue_pushchunked(taskid, &iovector) ) {
-        free(iovector.iov_base);
+    if (pending) {
+      /* Allocate a fresh output buffer */
+      iovector.iov_base = malloc( pending );
+      iovector.iov_len = pending;
+
+      if( !iovector.iov_base ) {
+        fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
+        deflateEnd(&strm);
         return mutex_bucket_unlock( bucket, 0 );
-    }
-    /* Allocate a fresh output buffer */
-    iovector.iov_base = malloc( pending );
-    iovector.iov_len = pending;
+      }
+      strm.next_out  = iovector.iov_base;
+      strm.avail_out = pending;
+      if( deflate( &strm, Z_FINISH ) < Z_OK )
+        fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
 
-    if( !iovector.iov_base ) {
-      fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
-      deflateEnd(&strm);
-      return mutex_bucket_unlock( bucket, 0 );
+      if( mutex_workqueue_pushchunked(taskid, &iovector) )
+        free(iovector.iov_base);
     }
-    strm.next_out  = iovector.iov_base;
-    strm.avail_out = pending;
-    if( deflate( &strm, Z_FINISH ) < Z_OK )
-      fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
-
-    if( mutex_workqueue_pushchunked(taskid, &iovector) )
-      free(iovector.iov_base);
   }
 
   deflateEnd(&strm);

ot_http.c

@@ -162,7 +162,7 @@ fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch
       header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size );
     else {
       if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) {
-        header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size );
+        header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size );
         cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER;
       } else
         header_size = sprintf( header, "%zx\r\n", size );
@@ -269,12 +269,12 @@ static const ot_keywords keywords_format[] =
     tai6464 t;
 #ifdef WANT_COMPRESSION_GZIP
     ws->request[ws->request_size] = 0;
-#ifdef WANT_COMPRESSION_GZIP_ALWAYS
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
     if( strstr( read_ptr - 1, "gzip" ) ) {
 #endif
       cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
       format |= TASK_FLAG_GZIP;
-#ifdef WANT_COMPRESSION_GZIP_ALWAYS
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
     }
 #endif
 #endif
@@ -337,11 +337,15 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w
 
 #ifdef WANT_COMPRESSION_GZIP
   ws->request[ws->request_size-1] = 0;
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
   if( strstr( ws->request, "gzip" ) ) {
+#endif
     cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
     format = TASK_FLAG_GZIP;
     stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip );
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
   } else
+#endif
 #endif
     stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip );
 

tests/testsuite2.sh

@@ -2,13 +2,21 @@
 
 while true; do
   request_string="GET /announce?info_hash=012345678901234567\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n"
-
-  echo $request_string
-  echo
-  echo $request_string | nc 23.23.23.237 6969 >/dev/null
-  echo
+$(printf %02X $(( $RANDOM & 0xff )) )\
+&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0"
+
+#  echo $request_string
+#  echo
+  printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C
+
+  request_string="GET /announce?info_hash=012345678901234567\
+$(printf %02X $(( $RANDOM & 0xff )) )\
+&ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0"
+  printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
+  printf "%s\n\n" "$request_string"
+
+  request_string="GET /scrape?info_hash=012345678901234567\
+$(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0"
+  printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
 
 done