Massively speed up dump by buffering output before write()

1 files changed, 55 insertions, 30 deletions

diff --git a/src/export/extract_version_1.c b/src/export/extract_version_1.c
index dc006e5..8ab3190 100644
--- a/src/export/extract_version_1.c
+++ b/src/export/extract_version_1.c
@@ -9,7 +9,7 @@
 #include "mystdlib.h"
 
 static uint8_t xlat_table[] = {
-  0x00, 0x09, 0x02, 0x03, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
+  0x00, 0x0a, 0x02, 0x03, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
   0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b,
   0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b,
   0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5f,
@@ -37,13 +37,17 @@ static uint8_t cp437_to_iso8859_1_table[] = {
 };
 
 static struct {
-  int    outfiles[15];
-  char * vorwahl;
-  char * ort;
-  char * zip;
+  int      outfiles[15];
+  uint8_t *outbuf[15];
+  size_t   outfill[15];
+  char *   vorwahl;
+  char     ort[1024];
+  size_t   ort_len;
+  char     zip[32];
+  size_t   zip_len;
 } g_state;
 
-static void decode_7bit_string( uint8_t const *source, uint8_t *dest )
+static uint8_t * decode_7bit_string( uint8_t const *source, uint8_t *dest )
 {
   uint16_t acc = 0, acc_bits = 0;
   while( 1 )
@@ -51,7 +55,7 @@ static void decode_7bit_string( uint8_t const *source, uint8_t *dest )
     acc = acc*256+*(source++); ++acc_bits;
 again:
     *(dest++) = xlat_table[ 0x7f & ( acc >> acc_bits ) ];
-    if( !dest[-1] ) return;
+    if( !dest[-1] ) return dest - 1;
     if( acc_bits == 7 ) { acc_bits = 0; goto again; }
   }
 }
@@ -64,28 +68,33 @@ static void split_to_files( uint8_t *entries, int num_entries )
   for( entry = 0; entry < num_entries; ++entry )
   {
     /* mimic flags from telefonbuch v3 */
-    write( g_state.outfiles[0], entry ? "02\n" : ( num_entries > 1 ? "01\n" : "00\n" ), 3 );
-    dprintf( g_state.outfiles[12], "%s\n", g_state.ort );
-    /* Only part of zip code, Zustellpostamt or eastern German code in column 5 (06_unknown) */
-    dprintf( g_state.outfiles[13], "%s\n", g_state.zip );
+    memcpy( g_state.outbuf[0 ] + g_state.outfill[0 ], entry ? "02\n" : ( num_entries > 1 ? "01\n" : "00\n" ), 3 ); g_state.outfill[0] += 3;
+    memcpy( g_state.outbuf[12] + g_state.outfill[12], g_state.ort, g_state.ort_len ); g_state.outfill[12] += g_state.ort_len;
+    memcpy( g_state.outbuf[13] + g_state.outfill[13], g_state.zip, g_state.zip_len ); g_state.outfill[13] += g_state.zip_len;
 
     for( column = 0; column < 11; ++column )
     {
-      char * end = strchr( e, '\t' );
-      if( end ) {
-        *end = '\n';
-        if( column == 9 && *e >= '0' && *e <= '9' ) write( g_state.outfiles[column+1], "0", 1 ); /* Augment Vorwahl */
-        write( g_state.outfiles[column+1], e, (size_t)(end - e + 1) );
-        e = end + 1;
-      } else
-        dprintf( g_state.outfiles[column+1], "%s\n", e);
+      char * end = strchr( e, '\n' );
+      if( !end ) {
+         fprintf( stderr, "Unexpected end of line in city: %s", g_state.ort );
+         /* fprintf( stderr, "Failing String, (%d of %d entries, column %d): ###%s\n", entry, num_entries, column, entries ); */
+         if( column ) for( ; column < 11; ++column) {
+           memcpy( g_state.outbuf[column+1] + g_state.outfill[column+1], "#\n", 2 );
+           g_state.outfill[column+1] += 2;
+         }
+         return;
+      }
+
+      if( column == 9 && *e >= '0' && *e <= '9' ) *--e = '0'; /* Augment Vorwahl */
+      memcpy( g_state.outbuf[column+1] + g_state.outfill[column+1], e, (size_t)(++end - e) ); g_state.outfill[column+1] += end - e;
+      e = end;
     }
   }
 }
 
 static void act_on_record( uint8_t *file, int flag, uint8_t *page, uint16_t record_off )
 {
-  uint8_t outbuf[8192], *out_dest = outbuf, *record;
+  uint8_t outbuf[8192*4], *out_dest = outbuf, *record;
   int     num_entries;
 
   if( record_off > 0x1fff )
@@ -112,7 +121,8 @@ static void act_on_record( uint8_t *file, int flag, uint8_t *page, uint16_t reco
     record += 2;
   }
 
-  decode_7bit_string( record, out_dest );
+  record = decode_7bit_string( record, out_dest );
+  *record++ = '\n'; *record = 0;
   split_to_files( outbuf, num_entries );
 }
 
@@ -134,24 +144,28 @@ static void act_on_page( uint8_t *file, uint8_t *page, int page_nr )
 
 static void act_on_file( uint8_t *file )
 {
-  int page, num_pages = *(uint16_t*)(file+0x40);
+  int i, page, num_pages = *(uint16_t*)(file+0x40);
   uint32_t num_records= *(uint32_t*)(file+0x42);
   char *gasse         = (char *)file + 0x8e;
   char *ort           = gasse + 1 + strlen(gasse);
   char *zip           = ort   + 1 + strlen(ort);
   char *vorwahl       = zip   + 1 + strlen(zip);
-  char ort_conv[1024]; int i;
 
-  for(i=0; ort[i]; ++i ) ort_conv[i] = (char)cp437_to_iso8859_1_table[((uint8_t*)ort)[i]]; ort_conv[i] = 0;
-  /* printf( "Working on a %04d page and %06d records file, city: %4s %-32s with prefix %s\n", num_pages, num_records, zip, ort_conv, vorwahl ); */
-  (void)num_records; /* silence warning about unused variable */
+  for(i=0; ort[i]; ++i )
+     g_state.ort[i] = (char)cp437_to_iso8859_1_table[((uint8_t*)ort)[i]];
+  g_state.ort[i++] = '\n';
+  g_state.ort[i]   = 0;
+  g_state.ort_len = i;
 
-  g_state.ort     = ort_conv;
-  g_state.zip     = zip;
+  g_state.zip_len = snprintf( g_state.zip, sizeof(g_state.zip), "%s\n", zip );
   g_state.vorwahl = vorwahl;
 
+  /* printf( "Working on a %04d page and %06d records file, city: %4s %-32s with prefix %s\n", num_pages, num_records, zip, ort, vorwahl ); */
+  (void)num_records; /* silence warning about unused variable */
+
   for( page = 0; page < num_pages; ++page )
     act_on_page( file, file + 0x800 + 0x2000 * page, page );
+
 }
 
 int main( )
@@ -164,6 +178,8 @@ int main( )
   {
     sprintf( filename, "%02d_unknown", i+1 );
     g_state.outfiles[i] = open( filename, O_WRONLY | O_APPEND | O_CREAT, 0644 );
+    g_state.outbuf[i]   = malloc(8192*4096);
+    g_state.outfill[i]  = 0;
   }
 
   while( fgets( filename, sizeof(filename), stdin ) ) {
@@ -172,10 +188,19 @@ int main( )
 
     act_on_file( f->addr );
     unmap_file( &f );
+
+    /* Write out results */
+    for( i=0; i<14; ++i ) {
+      /* if( g_state.outfill[i] > 1024*1024*6 ) printf( "Large: %s %zd\n", g_state.ort, g_state.outfill[i] ); */
+      write( g_state.outfiles[i], g_state.outbuf[i], g_state.outfill[i] );
+      g_state.outfill[i]  = 0;
+    }
   }
 
-  for( i=0; i<14; ++i )
-    close(  g_state.outfiles[i] );
+  for( i=0; i<14; ++i ) {
+    write( g_state.outfiles[i], g_state.outbuf[i], g_state.outfill[i] );
+    close( g_state.outfiles[i] );
+  }
 
   return 0;
 }