7 files changed, 68 insertions, 18 deletions

diff --git a/makecolumns.sh b/makecolumns.sh
index a505d31..cef88e0 100755
--- a/makecolumns.sh
+++ b/makecolumns.sh
@@ -373,6 +373,18 @@ handle_format_version_3() {
     rm ??_unknown
 }
 
+fix_broken_v4_file() {
+    name=$1
+    offset=$2
+    nullbytes=$3
+    gap=$4
+
+    head -c $(( offset - 1 )) file_${name} > file_${name}_
+    [ "${nullbytes}" -gt 0 ] && printf '\x00' >> file_${name}_
+    tail -c +$(( offset + gap )) file_${name} >> file_${name}_
+    mv file_${name}_ file_${name}
+}
+
 handle_format_version_4() {
     if [ -f "$1/branchcodes.tl" ]; then
       is_yp=true
@@ -392,6 +404,26 @@ handle_format_version_4() {
     printf "Extracting phonebook.db ... "
     extract_version_4 "$1/phonebook.db"
 
+    case $1 in
+    *2023_Q1*)
+        # Fixup broken file in 2023_Q1
+        fix_broken_v4_file 17F0A 0x01BB 1 93
+        ;;
+    *2023_Q3*)
+        # Fixup broken file in 2023_Q3
+        fix_broken_v4_file 15DB2 0x3D05 0 373
+        fix_broken_v4_file 15DBD 0x8CA8 0 391
+        fix_broken_v4_file 15DD3 0x72B7 0 120
+        fix_broken_v4_file 15E2B 0x0549 0 2051
+        fix_broken_v4_file 15EE6 0x0569 1 144
+        fix_broken_v4_file 15F28 0x2EF8 0 103
+        fix_broken_v4_file 15F3E 0x3C1C 0 164
+        fix_broken_v4_file 16348 0x0942 0 221
+        fix_broken_v4_file 16419 0x0CF8 0 140
+        fix_broken_v4_file 16471 0x0681 0 106
+        ;;
+    esac
+
     rows=`find . -name file_\* | wc -l`
     printf "done.\n"
 
diff --git a/postprocess.sh b/postprocess.sh
index bc70492..401a70f 100644
--- a/postprocess.sh
+++ b/postprocess.sh
@@ -26,4 +26,7 @@ fi
 
 cd work || exit 1
 
-for a in ????_Q?; do ./sort_plz $a; done
+for a in ????_Q?; do
+  printf "Sorting files from issue %s\n" $a
+  ./sort_plz $a
+done
diff --git a/shopping.txt b/shopping.txt
index c98df05..72ba624 100644
--- a/shopping.txt
+++ b/shopping.txt
@@ -3,8 +3,10 @@ https://www.buecher-nach-isbn.info/3-934073/
 https://www.buecher-nach-isbn.info/3-938913/
 
 Erscheinungsjahr        ISBN    Autor(en)       Titel
+1997    978-3-931424-46-4       Gelbe Seiten für Deutschland Herbst 1997
 1998    978-3-931424-50-3       Telefonbuch für Deutschland Frühjahr 98. CD- ROM für Windows 3.1/95/NT 3.5.1, Mac ab 7.5
 1999    978-3-931424-60-2       Telefonbuch für Deutschland Herbst 98. CD- ROM für Windows 3.1/95/98, Mac ab 7.5
+2000    978-3-931424-75-6       Gelbe Seiten für Deutschland Herbst 1999. CD- ROM für Windows ab 3.1/ MacOS ab 7.5. 3,8 Mio Kommunikationsnummern mit Branche und Branchenumfeld
 2001    978-3-931424-92-3       Gelbe Seiten Map & Route für Deutschland Herbst 2000. CD- ROM für Windows ab 95
   ''    978-3-931424-98-5       Das Telefonbuch für Deutschland Frühjahr 2001. 2 CD- ROMs für Windows 3.1/95/98/ NT 4.0/2000/ ME/ MacOS ab 7.5/ Linux
 2001    978-3-934073-04-3       Telefonbuch für Deutschland - Herbst 2001
diff --git a/src/export/extract_version_4.c b/src/export/extract_version_4.c
index 1dbaf40..e014e23 100644
--- a/src/export/extract_version_4.c
+++ b/src/export/extract_version_4.c
@@ -7,44 +7,56 @@
 #include "mystdlib.h"
 
 #define XORLEN (29)
+#define HUGEINBLOCK (128*1024)
 #define HUGEBLOCK (4*1024*1024)
 
 int main(int argc, char **argv) {
   unsigned const char xorkey [XORLEN] = "Just for Fun. Linus Torvalds.";
-  unsigned char input  [XORLEN];
+  unsigned char input  [HUGEINBLOCK];
   unsigned char output [HUGEBLOCK];
   char respath[32]; /* file_XXXXX\0 */
   int zres = 0, filenum = 0, resfile;
-  size_t i, offs = 0, reported = 0;
+  size_t offs = 0, reported = 0;
   ssize_t temp = 0;
   MAP in;
 
   if( argc != 2 ) exit(111);
   in = map_file( argv[1], 1 );
 
-  z_stream z; memset( &z, 0, sizeof(z));
+  z_stream z;
 
   while( offs < in->size ) {
-    size_t inlen = offs + XORLEN < in->size ? XORLEN : in->size - offs;
-    for( i=0; i<inlen; ++i ) input[i] = in->addr[offs+i] ^ xorkey[i];
+    /* std::min(remain, HUGEINBLOCK) */
+    size_t inlen = in->size - offs;
+    if (inlen > sizeof(input))
+        inlen = sizeof(input);
+
+    /* Copy in block large enough */
+    memcpy(input, in->addr + offs, inlen);
+
+    /* De-"crypt" */
+    for (size_t i = 0; i < sizeof(xorkey); ++i )
+      input[i] ^= xorkey[i];
+
+    /* Prepare decompression struct */
+    memset( &z, 0, sizeof(z));
     z.next_in  = input;  z.avail_in = inlen;
     z.next_out = output; z.avail_out = HUGEBLOCK;
-    inflateInit( &z ); zres = inflate( &z, Z_NO_FLUSH );
-    if( (zres != Z_OK) && (zres != Z_STREAM_END) )
-      goto error_continue;
+    inflateInit( &z );
+    zres = Z_OK;
 
-    z.next_in  = in->addr + offs + inlen;
-    z.avail_in = (unsigned int)(in->size - offs - inlen);
-    while( zres == Z_OK ) zres = inflate( &z, Z_NO_FLUSH );
+    while( zres == Z_OK )
+      zres = inflate( &z, Z_NO_FLUSH );
 
     if( zres != Z_STREAM_END ) {
-error_continue:
-      inflateEnd(&z); memset( &z, 0, sizeof(z));
+      inflateEnd(&z);
       offs++;
       continue;
     }
 
-    sprintf( respath, "file_%05X", filenum++ );
+    // fprintf( stderr, "%08X\n", (unsigned int)(offs));
+    //old_offs = offs;
+    snprintf( respath, sizeof(respath), "file_%05X", filenum++ );
 
     resfile = open( respath, O_RDWR | O_CREAT, 0644 );
     if( resfile < 0 ) {
diff --git a/src/postprocess/merge_entries.c b/src/postprocess/merge_entries.c
index 1dd7d50..aed0aec 100644
--- a/src/postprocess/merge_entries.c
+++ b/src/postprocess/merge_entries.c
@@ -26,7 +26,7 @@ const char *g_year_map[] = {
 "1992_Q2", "1995_Q0", "1996_Q0", "1996_Q1", "1997_Q1", "1997_Q3", "1998_Q1", "1998_Q3", "1999_Q1", "1999_Q3", "2000_Q1", "2000_Q3", "2001_Q1", "2001_Q2", "2001_Q3", "2001_Q4", "2002_Q1",
 "2002_Q3", "2003_Q1", "2003_Q3", "2004_Q1", "2004_Q3", "2005_Q1", "2005_Q3", "2006_Q1", "2006_Q3", "2007_Q1", "2007_Q3", "2008_Q1", "2008_Q3", "2009_Q1", "2009_Q3", "2010_Q1", "2010_Q3",
 "2011_Q1", "2011_Q3", "2012_Q1", "2012_Q3", "2013_Q1", "2013_Q3", "2014_Q1", "2014_Q3", "2015_Q1", "2015_Q3", "2016_Q1", "2016_Q3", "2017_Q1", "2017_Q3", "2018_Q1", "2018_Q3", "2019_Q1",
-0
+"2019_Q3", "2020_Q1", "2020_Q3", "2021_Q1", "2021_Q3", "2022_Q1", "2022_Q3", "2023_Q1", "2023_Q3", "2024_Q1", "2024_Q3", "2025_Q1", NULL
 };
 
 static int year_to_offset(const char *year) {
diff --git a/src/postprocess/postgres.sql b/src/postprocess/postgres.sql
index 2d89d1c..3c891d4 100644
--- a/src/postprocess/postgres.sql
+++ b/src/postprocess/postgres.sql
@@ -74,7 +74,7 @@ CREATE INDEX idx_strasse    ON  table_strasse    USING btree (value, telefonbuch
 CREATE INDEX idx_hausnummer ON  table_hausnummer USING btree (value, telefonbuch_id, offs);
 CREATE INDEX idx_zip        ON  table_zip        USING btree (value, telefonbuch_id, offs);
 CREATE INDEX idx_ort        ON  table_ort        USING btree (value, telefonbuch_id, offs);
-CREATE INDEX idx_verweise   ON  table_verweise   USING btree (value, telefonbuch_id, offs);
+CREATE INDEX idx_verweise   ON  table_verweise   USING btree (value, telefonbuch_id, offs) WHERE LENGTH(value) < 256;
 CREATE INDEX idx_vorwahl    ON  table_vorwahl    USING btree (value, telefonbuch_id, offs);
 CREATE INDEX idx_rufnummer  ON  table_rufnummer  USING btree (value, telefonbuch_id, offs);
 CREATE INDEX idx_web        ON  table_web        USING btree (value, telefonbuch_id, offs);
diff --git a/src/postprocess/sort_plz.c b/src/postprocess/sort_plz.c
index dc0b222..9c67ab7 100644
--- a/src/postprocess/sort_plz.c
+++ b/src/postprocess/sort_plz.c
@@ -13,6 +13,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <stdint.h>
 #include <errno.h>
 #include <unistd.h>
 #include <err.h>
@@ -84,7 +85,7 @@ int main(int argc, char **args) {
   char *input = malloc(1024);
   size_t input_size = 1024;
 
-  if (argc != 1) exit(1);
+  if (argc != 2) exit(1);
 
   /* First open all input files */
   for (i=F_01; i<F_COUNT; ++i) {