summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDirk Engling <erdgeist@erdgeist.org>2015-04-29 12:44:47 +0200
committerDirk Engling <erdgeist@erdgeist.org>2015-04-29 12:44:47 +0200
commit43a5ac139b552b23de78434a8ee3df8fc6651b38 (patch)
tree759f5b2a10719d1a12d5ebf0279b4e193506141b
parent18c51711a08db1a13f7829638295e062b90d8601 (diff)
We have a new format between the former version 1 and 2. So shift version numbers. Add README
-rw-r--r--Makefile14
-rw-r--r--README8
-rwxr-xr-xmakecolumns.sh34
-rw-r--r--src/export/extract_version_15.c404
-rw-r--r--src/export/extract_version_2.c488
-rw-r--r--src/export/extract_version_3.c170
-rw-r--r--src/export/extract_version_4.c70
-rw-r--r--src/export/split_version_3.c (renamed from src/export/split_version_2.c)0
8 files changed, 598 insertions, 590 deletions
diff --git a/Makefile b/Makefile
index 5ad8100..01507ae 100644
--- a/Makefile
+++ b/Makefile
@@ -1,19 +1,19 @@
1BINARIES=bin/extract_version_1 bin/extract_version_2 bin/extract_version_3 bin/split_version_2 bin/map_coords bin/convert_coords 1BINARIES=bin/extract_version_1 bin/extract_version_3 bin/extract_version_4 bin/split_version_3 bin/map_coords bin/convert_coords
2CFLAGS += -W -Wall -Wextra -O3 # -Weverything -Wno-cast-align -Wno-padded 2CFLAGS += -W -Wall -Wextra -O3 # -Weverything -Wno-cast-align -Wno-padded
3 3
4all: $(BINARIES) 4all: $(BINARIES)
5 5
6bin/extract_version_3: src/export/extract_version_3.c src/export/mystdlib.c 6bin/extract_version_4: src/export/extract_version_4.c src/export/mystdlib.c
7 $(CC) $(CFLAGS) -o $@ src/export/extract_version_3.c src/export/mystdlib.c -lz 7 $(CC) $(CFLAGS) -o $@ src/export/extract_version_4.c src/export/mystdlib.c -lz
8 8
9bin/extract_version_2: src/export/extract_version_2.c src/export/mystdlib.c 9bin/extract_version_3: src/export/extract_version_3.c src/export/mystdlib.c
10 $(CC) $(CFLAGS) -o $@ src/export/extract_version_2.c src/export/mystdlib.c 10 $(CC) $(CFLAGS) -o $@ src/export/extract_version_3.c src/export/mystdlib.c
11 11
12bin/extract_version_1: src/export/extract_version_1.c src/export/mystdlib.c 12bin/extract_version_1: src/export/extract_version_1.c src/export/mystdlib.c
13 $(CC) $(CFLAGS) -o $@ src/export/extract_version_1.c src/export/mystdlib.c 13 $(CC) $(CFLAGS) -o $@ src/export/extract_version_1.c src/export/mystdlib.c
14 14
15bin/split_version_2: src/export/split_version_2.c 15bin/split_version_3: src/export/split_version_3.c
16 $(CC) $(CFLAGS) -o $@ src/export/split_version_2.c 16 $(CC) $(CFLAGS) -o $@ src/export/split_version_3.c
17 17
18bin/map_coords: src/export/map_coords.c src/export/mystdlib.c 18bin/map_coords: src/export/map_coords.c src/export/mystdlib.c
19 $(CC) $(CFLAGS) -o $@ src/export/map_coords.c src/export/mystdlib.c 19 $(CC) $(CFLAGS) -o $@ src/export/map_coords.c src/export/mystdlib.c
diff --git a/README b/README
new file mode 100644
index 0000000..824bb38
--- /dev/null
+++ b/README
@@ -0,0 +1,8 @@
1This project can currently dump and (partially) normalize white pages from Deutsche Telekom's CD and DVDs.
2
3The on-disc-data currently comes in four flavours (see https://erdgeist.org/posts/2008/datenmessie.html)
4
5version 1) Teleauskunft 1188 from 1992, (April-June)
6version 2) Teleauskunft 1188 Telefon-Teilnehmer, Oktober 1995 / Telefon-Teilnehmer Gesamtausgabe from 1995/1996
7version 3) Telefonbuch für Deutschland, Version 1.0 1996 through DasTelefonbuch, Deutschland, Herbst 2003
8version 4) DasTelefonbuch, Map&Route, Frühjahr 2004 until now
diff --git a/makecolumns.sh b/makecolumns.sh
index b60cfd6..5d2f1aa 100755
--- a/makecolumns.sh
+++ b/makecolumns.sh
@@ -29,9 +29,9 @@ main() {
29 cd work/`basename "${1#white_}"` || exit 1 29 cd work/`basename "${1#white_}"` || exit 1
30 30
31 if [ -f "$1/phonebook.db" ]; then 31 if [ -f "$1/phonebook.db" ]; then
32 handle_format_version_3 "${1}" 32 handle_format_version_4 "${1}"
33 elif [ -f ${1}/[Dd][Aa][Tt]/[Tt][Ee][Ii][Ll][Nn].[Dd][Aa][Tt] ]; then 33 elif [ -f ${1}/[Dd][Aa][Tt]/[Tt][Ee][Ii][Ll][Nn].[Dd][Aa][Tt] ]; then
34 handle_format_version_2 "${1}" 34 handle_format_version_3 "${1}"
35 elif [ -n "`find "${1}" -name dpr00000.005 -ls -quit`" ]; then 35 elif [ -n "`find "${1}" -name dpr00000.005 -ls -quit`" ]; then
36 handle_format_version_1 "${1}" 36 handle_format_version_1 "${1}"
37 else 37 else
@@ -40,9 +40,9 @@ main() {
40 cd ../.. 40 cd ../..
41} 41}
42 42
43do_decompress_version_2() { 43do_decompress_version_3() {
44 printf "Extracting $2 chunks ... " 44 printf "Extracting $2 chunks ... "
45 extract_version_2 "${1}" 45 extract_version_3 "${1}"
46 printf "done.\n" 46 printf "done.\n"
47 47
48 printf "Decompressing $2 chunks ... " 48 printf "Decompressing $2 chunks ... "
@@ -57,10 +57,10 @@ do_decompress_version_2() {
57 printf "done.\n" 57 printf "done.\n"
58} 58}
59 59
60do_processfile_version_2() { 60do_processfile_version_3() {
61 working_on=`basename ${1}` 61 working_on=`basename ${1}`
62 mkdir $working_on && cd ${working_on} 62 mkdir $working_on && cd ${working_on}
63 do_decompress_version_2 "${1}" "${2}" 63 do_decompress_version_3 "${1}" "${2}"
64 cd .. 64 cd ..
65 65
66 printf "Combining $2 into single file ... " 66 printf "Combining $2 into single file ... "
@@ -109,10 +109,10 @@ handle_format_version_1() {
109} 109}
110 110
111 111
112handle_format_version_2() { 112handle_format_version_3() {
113 echo "Working on $1. Detected pre-2004 Telefonbuch version." 113 echo "Working on $1. Detected pre-2004 Telefonbuch version."
114 # Extract teiln.dat 114 # Extract teiln.dat
115 do_decompress_version_2 $1/[Dd][Aa][Tt]/[Tt][Ee][Ii][Ll][Nn].[Dd][Aa][Tt] "teiln.dat" 115 do_decompress_version_3 $1/[Dd][Aa][Tt]/[Tt][Ee][Ii][Ll][Nn].[Dd][Aa][Tt] "teiln.dat"
116 116
117 # See how long each filename is 117 # See how long each filename is
118 export filename_len=$(( `ls | head -n 1 | wc -c` - 1 )) 118 export filename_len=$(( `ls | head -n 1 | wc -c` - 1 ))
@@ -140,7 +140,7 @@ handle_format_version_2() {
140 140
141 # Now loop over all files and dump them 141 # Now loop over all files and dump them
142 printf "Splitting decompressed nname chunks into their columns ... " 142 printf "Splitting decompressed nname chunks into their columns ... "
143 jot -w %0${filename_len}d - ${nname_file} $(( number_of_files - 1 )) 3 | split_version_2 1 1 143 jot -w %0${filename_len}d - ${nname_file} $(( number_of_files - 1 )) 3 | split_version_3 1 1
144# set -- `hexdump -n 8 -v -e '" " 1/4 "%u"' ${file}` 144# set -- `hexdump -n 8 -v -e '" " 1/4 "%u"' ${file}`
145# tail -c +$(( $2 + 1 )) ${file} 145# tail -c +$(( $2 + 1 )) ${file}
146# done | tr '\n\0' '\t\n' > 01_02_Flags_Nachname 146# done | tr '\n\0' '\t\n' > 01_02_Flags_Nachname
@@ -154,7 +154,7 @@ handle_format_version_2() {
154 printf "done.\n" 154 printf "done.\n"
155 155
156 printf "Splitting decompress table file chunks into their columns ... " 156 printf "Splitting decompress table file chunks into their columns ... "
157 jot -w %0${filename_len}d - ${table_file} $(( number_of_files - 1 )) 3 | split_version_2 4 0 157 jot -w %0${filename_len}d - ${table_file} $(( number_of_files - 1 )) 3 | split_version_3 4 0
158# for file in `jot -w %0${filename_len}d - ${table_file} $(( number_of_files - 1 )) 3`; do 158# for file in `jot -w %0${filename_len}d - ${table_file} $(( number_of_files - 1 )) 3`; do
159# # Offset into first table entry tells us how many 159# # Offset into first table entry tells us how many
160# # fields are in table file 160# # fields are in table file
@@ -198,7 +198,7 @@ handle_format_version_2() {
198 # If street names come in an extra file, extract 198 # If street names come in an extra file, extract
199 # street names first 199 # street names first
200 streets=$1/[Dd][Aa][Tt]/[Ss][Tt][Rr][Aa][Ss][Ss][Ee][Nn].[Dd][Aa][Tt] 200 streets=$1/[Dd][Aa][Tt]/[Ss][Tt][Rr][Aa][Ss][Ss][Ee][Nn].[Dd][Aa][Tt]
201 [ -f ${streets} ] && do_processfile_version_2 ${streets} "street name" 99_Strassenname convert_zeros 201 [ -f ${streets} ] && do_processfile_version_3 ${streets} "street name" 99_Strassenname convert_zeros
202 202
203 # extract street names if 07_unknown contains street indexes 203 # extract street names if 07_unknown contains street indexes
204 # instead of street names 204 # instead of street names
@@ -213,7 +213,7 @@ handle_format_version_2() {
213 213
214 karto=$1/[Dd][Aa][Tt]/[Kk][Aa][Rr][Tt][Oo].[Dd][Aa][Tt] 214 karto=$1/[Dd][Aa][Tt]/[Kk][Aa][Rr][Tt][Oo].[Dd][Aa][Tt]
215 if [ -f ${karto} ]; then 215 if [ -f ${karto} ]; then
216 do_processfile_version_2 ${karto} "geo coordinates" 90_Geokoordinaten_hnr_raw 216 do_processfile_version_3 ${karto} "geo coordinates" 90_Geokoordinaten_hnr_raw
217 217
218 printf "Looking up geo coordinates for each phonebook entry ... " 218 printf "Looking up geo coordinates for each phonebook entry ... "
219 tr '\0' '\n' < 90_Geokoordinaten_hnr_raw | tr ';' '\t' | cut -f "1,2,3,4,6,7" | tr '\n' '\0' > 90_Geokoordinaten_hnr 219 tr '\0' '\n' < 90_Geokoordinaten_hnr_raw | tr ';' '\t' | cut -f "1,2,3,4,6,7" | tr '\n' '\0' > 90_Geokoordinaten_hnr
@@ -223,17 +223,17 @@ handle_format_version_2() {
223 fi 223 fi
224} 224}
225 225
226handle_format_version_3() { 226handle_format_version_4() {
227 echo "Working on $1. Detected post-2003 Telefonbuch version." 227 echo "Working on $1. Detected post-2003 Telefonbuch version."
228 printf "Extracting street names ... " 228 printf "Extracting street names ... "
229 extract_version_3 $1/streets.tl 229 extract_version_4 $1/streets.tl
230 230
231 cat file_* | tr '\n\0' '\t\n' > 99_Strassenname 231 cat file_* | tr '\n\0' '\t\n' > 99_Strassenname
232 rm file_* 232 rm file_*
233 printf "done.\n" 233 printf "done.\n"
234 234
235 printf "Extracting phonebook.db ... " 235 printf "Extracting phonebook.db ... "
236 extract_version_3 $1/phonebook.db 236 extract_version_4 $1/phonebook.db
237 237
238 rows=`find . -name file_\* | wc -l` 238 rows=`find . -name file_\* | wc -l`
239 printf "done.\n" 239 printf "done.\n"
@@ -277,7 +277,7 @@ handle_format_version_3() {
277 277
278 if [ -f $1/zip-streets-hn-geo.tl ]; then 278 if [ -f $1/zip-streets-hn-geo.tl ]; then
279 printf "Extracting geo coordinates (precision: house number) ... " 279 printf "Extracting geo coordinates (precision: house number) ... "
280 extract_version_3 $1/zip-streets-hn-geo.tl 280 extract_version_4 $1/zip-streets-hn-geo.tl
281 cat file_* > 90_Geokoordinaten_hnr 281 cat file_* > 90_Geokoordinaten_hnr
282 printf "done.\n" 282 printf "done.\n"
283 printf "Looking up geo coordinates for each phonebook entry ... " 283 printf "Looking up geo coordinates for each phonebook entry ... "
@@ -285,7 +285,7 @@ handle_format_version_3() {
285 printf "done.\n" 285 printf "done.\n"
286 elif [ -f $1/zip-streets-geo.tl ]; then 286 elif [ -f $1/zip-streets-geo.tl ]; then
287 printf "Extracting geo coordinates (precision: street) ... " 287 printf "Extracting geo coordinates (precision: street) ... "
288 extract_version_3 $1/zip-streets-geo.tl 288 extract_version_4 $1/zip-streets-geo.tl
289 cat file_* > 91_Geokoordinaten_str 289 cat file_* > 91_Geokoordinaten_str
290 printf "done.\n" 290 printf "done.\n"
291 printf "Looking up geo coordinates for each phonebook entry ... " 291 printf "Looking up geo coordinates for each phonebook entry ... "
diff --git a/src/export/extract_version_15.c b/src/export/extract_version_15.c
deleted file mode 100644
index 68b05af..0000000
--- a/src/export/extract_version_15.c
+++ /dev/null
@@ -1,404 +0,0 @@
1/*
2 This decompressor uses modified code from zlib. Below is it's original copyright
3 notice.
4
5 Copyright (C) 2003, 2012 Mark Adler
6 version 1.2, 24 Oct 2012
7
8 This software is provided 'as-is', without any express or implied
9 warranty. In no event will the author be held liable for any damages
10 arising from the use of this software.
11
12 Permission is granted to anyone to use this software for any purpose,
13 including commercial applications, and to alter it and redistribute it
14 freely, subject to the following restrictions:
15
16 1. The origin of this software must not be misrepresented; you must not
17 claim that you wrote the original software. If you use this software
18 in a product, an acknowledgment in the product documentation would be
19 appreciated but is not required.
20 2. Altered source versions must be plainly marked as such, and must not be
21 misrepresented as being the original software.
22 3. This notice may not be removed or altered from any source distribution.
23
24 Mark Adler madler@alumni.caltech.edu
25 */
26
27#include <stdlib.h>
28#include <stdio.h>
29#include <stdint.h>
30#include "mystdlib.h"
31#include <string.h>
32
33/*
34uint8_t table[] = {
35 0x00, 0x39, 0x20, 0x15, 0x24, 0x04, 0x61, 0x80, 0xc4, 0xc0, 0x1f, 0xa4, 0x04, 0xca, 0x40, 0xb4,
36 0x08, 0x19, 0xe7, 0x03, 0xab, 0xe0, 0x83, 0xac, 0x12, 0x92, 0x82, 0x95, 0x10, 0x5b, 0x24, 0x0c,
37 0x75, 0x81, 0xaf, 0xe0, 0x3a, 0x24, 0x07, 0xc7, 0xc1, 0x09, 0x88, 0x23, 0x55, 0x84, 0xac, 0x50,
38 0x9d, 0xc6, 0x14, 0xc3, 0x82, 0xb9, 0x30, 0x5b, 0x54, 0x0b, 0xf1, 0x81, 0x8e, 0xbc, 0x33, 0xce,
39 0x86, 0xb9, 0xa0, 0xdf, 0xae, 0x1d, 0x04, 0x03, 0xc0, 0xf8, 0x7c, 0x13, 0x0f, 0xf0, 0x22, 0x0b,
40 0xa8, 0x43, 0x3e, 0x88, 0xa0, 0xd1, 0x1b, 0x48, 0x24, 0x61, 0x84, 0xad, 0xb0, 0x99, 0xe0, 0x13,
41 0xc0, 0x62, 0x87, 0xdc, 0x52, 0xba, 0x8a, 0x8f, 0x31, 0x59, 0x0e, 0x2c, 0x16, 0x85, 0xa0, 0x38,
42 0xb7, 0x8a, 0x17, 0x66, 0xc2, 0xfb, 0x50, 0x61, 0x3f, 0x8c, 0x5f, 0x01, 0x93, 0x76, 0x33, 0x6f,
43 0x46, 0x8e, 0x18, 0xd5, 0x6f, 0x1b, 0x1e, 0x83, 0x71, 0x6c, 0x6f, 0xfc, 0x0e, 0x3d, 0x91, 0xcf,
44 0x08, 0x3a, 0xc4, 0x87, 0x75, 0x80, 0xf2, 0x23, 0x1e, 0xb4, 0x83, 0xe4, 0xe0, 0x7e, 0x6d, 0x10,
45 0x06, 0x22, 0x08, 0x5a, 0x41, 0xf8, 0x48, 0x5b, 0x11, 0x0e, 0xce, 0x22, 0x46, 0x44, 0x56, 0x84,
46 0x8c, 0xc9, 0x91, 0xdc, 0x22, 0x44, 0x26, 0x49, 0x9f, 0x49, 0x56, 0x71, 0x2f, 0x18, 0x26, 0x6d,
47 0xa4, 0xde, 0xdc, 0x9e, 0x02, 0x94, 0x04, 0xc2, 0x89, 0x38, 0x52, 0x31, 0x8a, 0x68, 0x39, 0x51,
48 0x64, 0x2a, 0xb0, 0x45, 0x65, 0x80, 0xae, 0xbf, 0x16, 0x19, 0x12, 0xcb, 0x48, 0x5a, 0x6b, 0xcb,
49 0x6a, 0x11, 0x70, 0xa3, 0x2e, 0x80, 0x85, 0xdd, 0xf0, 0xbd, 0x7a, 0x97, 0xe5, 0xb3, 0x03, 0xbc,
50 0x61, 0x67, 0xcc, 0x49, 0xe1, 0x8d, 0x11, 0x32, 0x28, 0x66, 0x53, 0xd8, 0xcc, 0x2b, 0x19, 0xbd,
51 0x93, 0x3e, 0x54, 0x68, 0xae, 0x4d, 0x35, 0xe1, 0xaa, 0xbe, 0x35, 0xd5, 0x46, 0xc8, 0x30, 0xda,
52 0xd0, 0x9b, 0x95, 0x73, 0x7a, 0xc8, 0x70, 0x43, 0x4e, 0x23, 0x19, 0xc8, 0x41, 0x39, 0x76, 0x07,
53 0x3d, 0xb8, 0xe9, 0x79, 0x9d, 0x65, 0xe3, 0xb3, 0x94, 0x77, 0x47, 0xcf, 0x04, 0xf1, 0xe3, 0xfc,
54 0x3c, 0xf0, 0x27, 0xac, 0x30, 0xf7, 0x4b, 0x1f, 0x25, 0x03, 0xec, 0x0a, 0x7e, 0x65, 0xcf, 0xe8,
55 0x1a, 0x00, 0xa7, 0x40, 0x8d, 0xc8, 0x21, 0x45, 0x05, 0xf5, 0xa0, 0xf7, 0x34, 0x25, 0xd0, 0x85,
56 0x9a, 0x50, 0xcf, 0x02, 0x1d, 0xd2, 0x44, 0x2c, 0x28, 0x93, 0xdd, 0x14, 0x36, 0x22, 0xbf, 0x24,
57 0x5e, 0x86, 0x8c, 0xa9, 0x91, 0xb1, 0x8a, 0x39, 0xd3, 0x47, 0xab, 0x29, 0x03, 0x29, 0x22, 0x28,
58 0xa4, 0x7b, 0xc4, 0x96, 0x68, 0x93, 0xb9, 0x12, 0x92, 0xfa, 0x55, 0xd2, 0x4b, 0x2d, 0x29, 0x74,
59 0x05, 0x30, 0x38, 0xa6, 0x40, 0xd4, 0xcf, 0x6e, 0x9a, 0xcc, 0x13, 0x74, 0xda, 0x72, 0x0a, 0x4e,
60 0xb3, 0x29, 0xe4, 0xa9, 0x3e, 0x60, 0xa8, 0x04, 0xa5, 0x07, 0xce, 0xa1, 0xe7, 0xc0
61};
62*/
63
64#define MAXBITS 13 /* maximum code length */
65#define MAXWIN 8192 /* maximum window size */
66
67struct state {
68 uint8_t *in; /* next input location */
69 uint8_t *out; /* output buffer and sliding window */
70
71 unsigned left; /* available input at in */
72 int bitbuf; /* bit buffer */
73 int bitcnt; /* number of bits in bit buffer */
74
75 unsigned next; /* index of next write location in out[] */
76 int first; /* true to check distances (for first 4K) */
77};
78
79static int bits(struct state *s, int need)
80{
81 int val; /* bit accumulator */
82
83 /* load at least need bits into val */
84 val = s->bitbuf;
85 while (s->bitcnt < need) {
86 val |= (int)(*(s->in)++) << s->bitcnt; /* load eight bits */
87 s->left--;
88 s->bitcnt += 8;
89 }
90
91 /* drop need bits and update buffer, always zero to seven bits left */
92 s->bitbuf = val >> need;
93 s->bitcnt -= need;
94
95 /* return need bits, zeroing the bits above that */
96 return val & ((1 << need) - 1);
97}
98
99struct huffman {
100 short *count; /* number of symbols of each length */
101 short *symbol; /* canonically ordered symbols */
102};
103
104static int decode(struct state *s, struct huffman *h)
105{
106 int len; /* current number of bits in code */
107 int code; /* len bits being decoded */
108 int first; /* first code of length len */
109 int count; /* number of codes of length len */
110 int index; /* index of first code of length len in symbol table */
111 int bitbuf; /* bits from stream */
112 int left; /* bits left in next or left to process */
113 short *next; /* next number of codes */
114
115 bitbuf = s->bitbuf;
116 left = s->bitcnt;
117 code = first = index = 0;
118 len = 1;
119 next = h->count + 1;
120 while (1) {
121 while (left--) {
122 code |= (bitbuf & 1) ^ 1; /* invert code */
123 bitbuf >>= 1;
124 count = *next++;
125 if (code < first + count) { /* if length len, return symbol */
126 s->bitbuf = bitbuf;
127 s->bitcnt = (s->bitcnt - len) & 7;
128 return h->symbol[index + (code - first)];
129 }
130 index += count; /* else update for next length */
131 first += count;
132 first <<= 1;
133 code <<= 1;
134 len++;
135 }
136 left = (MAXBITS+1) - len;
137 if (left == 0) break;
138 bitbuf = *(s->in)++;
139 s->left--;
140 if (left > 8) left = 8;
141 }
142 return -9; /* ran out of codes */
143}
144
145static int construct(struct huffman *h, const unsigned char *rep, int n)
146{
147 int symbol; /* current symbol when stepping through length[] */
148 int len; /* current length when stepping through h->count[] */
149 int left; /* number of possible codes left of current length */
150 short offs[MAXBITS+1]; /* offsets in symbol table for each length */
151 short length[256]; /* code lengths */
152
153 /* convert compact repeat counts into symbol bit length list */
154 symbol = 0;
155 do {
156 len = *rep++;
157 left = (len >> 4) + 1;
158 len &= 15;
159 do {
160 length[symbol++] = len;
161 } while (--left);
162 } while (--n);
163 n = symbol;
164
165 /* count number of codes of each length */
166 for (len = 0; len <= MAXBITS; len++)
167 h->count[len] = 0;
168 for (symbol = 0; symbol < n; symbol++)
169 (h->count[length[symbol]])++; /* assumes lengths are within bounds */
170 if (h->count[0] == n) /* no codes! */
171 return 0; /* complete, but decode() will fail */
172
173 /* check for an over-subscribed or incomplete set of lengths */
174 left = 1; /* one possible code of zero length */
175 for (len = 1; len <= MAXBITS; len++) {
176 left <<= 1; /* one more bit, double codes left */
177 left -= h->count[len]; /* deduct count from possible codes */
178 if (left < 0) return left; /* over-subscribed--return negative */
179 } /* left > 0 means incomplete */
180
181 /* generate offsets into symbol table for each length for sorting */
182 offs[1] = 0;
183 for (len = 1; len < MAXBITS; len++)
184 offs[len + 1] = offs[len] + h->count[len];
185
186 /*
187 * put symbols in table sorted by length, by symbol order within each
188 * length
189 */
190 for (symbol = 0; symbol < n; symbol++)
191 if (length[symbol] != 0)
192 h->symbol[offs[length[symbol]]++] = symbol;
193
194 /* return zero for complete set, positive for incomplete set */
195 return left;
196}
197
198/*
199 * Decode PKWare Compression Library stream.
200 *
201 * Format notes:
202 *
203 * - First byte is 0 if literals are uncoded or 1 if they are coded. Second
204 * byte is 4, 5, or 6 for the number of extra bits in the distance code.
205 * This is the base-2 logarithm of the dictionary size minus six.
206 *
207 * - Compressed data is a combination of literals and length/distance pairs
208 * terminated by an end code. Literals are either Huffman coded or
209 * uncoded bytes. A length/distance pair is a coded length followed by a
210 * coded distance to represent a string that occurs earlier in the
211 * uncompressed data that occurs again at the current location.
212 *
213 * - A bit preceding a literal or length/distance pair indicates which comes
214 * next, 0 for literals, 1 for length/distance.
215 *
216 * - If literals are uncoded, then the next eight bits are the literal, in the
217 * normal bit order in th stream, i.e. no bit-reversal is needed. Similarly,
218 * no bit reversal is needed for either the length extra bits or the distance
219 * extra bits.
220 *
221 * - Literal bytes are simply written to the output. A length/distance pair is
222 * an instruction to copy previously uncompressed bytes to the output. The
223 * copy is from distance bytes back in the output stream, copying for length
224 * bytes.
225 *
226 * - Distances pointing before the beginning of the output data are not
227 * permitted.
228 *
229 * - Overlapped copies, where the length is greater than the distance, are
230 * allowed and common. For example, a distance of one and a length of 518
231 * simply copies the last byte 518 times. A distance of four and a length of
232 * twelve copies the last four bytes three times. A simple forward copy
233 * ignoring whether the length is greater than the distance or not implements
234 * this correctly.
235 */
236static int decomp(struct state *s)
237{
238 int lit; /* true if literals are coded */
239 int dict; /* log2(dictionary size) - 6 */
240 int symbol; /* decoded symbol, extra bits for distance */
241 int len; /* length for copy */
242 unsigned dist; /* distance for copy */
243 int copy; /* copy counter */
244 unsigned char *from, *to; /* copy pointers */
245 static int virgin = 1; /* build tables once */
246 static short litcnt[MAXBITS+1], litsym[256]; /* litcode memory */
247 static short lencnt[MAXBITS+1], lensym[16]; /* lencode memory */
248 static short distcnt[MAXBITS+1], distsym[64]; /* distcode memory */
249 static struct huffman litcode = {litcnt, litsym}; /* length code */
250 static struct huffman lencode = {lencnt, lensym}; /* length code */
251 static struct huffman distcode = {distcnt, distsym};/* distance code */
252 /* bit lengths of literal codes */
253 static const unsigned char litlen[] = {
254 11, 124, 8, 7, 28, 7, 188, 13, 76, 4, 10, 8, 12, 10, 12, 10, 8, 23, 8,
255 9, 7, 6, 7, 8, 7, 6, 55, 8, 23, 24, 12, 11, 7, 9, 11, 12, 6, 7, 22, 5,
256 7, 24, 6, 11, 9, 6, 7, 22, 7, 11, 38, 7, 9, 8, 25, 11, 8, 11, 9, 12,
257 8, 12, 5, 38, 5, 38, 5, 11, 7, 5, 6, 21, 6, 10, 53, 8, 7, 24, 10, 27,
258 44, 253, 253, 253, 252, 252, 252, 13, 12, 45, 12, 45, 12, 61, 12, 45,
259 44, 173};
260 /* bit lengths of length codes 0..15 */
261 static const unsigned char lenlen[] = {2, 35, 36, 53, 38, 23};
262 /* bit lengths of distance codes 0..63 */
263 static const unsigned char distlen[] = {2, 20, 53, 230, 247, 151, 248};
264 static const short base[16] = { /* base for length codes */
265 3, 2, 4, 5, 6, 7, 8, 9, 10, 12, 16, 24, 40, 72, 136, 264};
266 static const char extra[16] = { /* extra bits for length codes */
267 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8};
268
269 /* set up decoding tables (once--might not be thread-safe) */
270 if (virgin) {
271 construct(&litcode, litlen, sizeof(litlen));
272 construct(&lencode, lenlen, sizeof(lenlen));
273 construct(&distcode, distlen, sizeof(distlen));
274 virgin = 0;
275 }
276
277 /* read header */
278 lit = bits(s, 8);
279 if (lit > 1) return -1;
280 dict = bits(s, 8);
281 if (dict < 4 || dict > 6) return -2;
282
283 /* decode literals and length/distance pairs */
284 do {
285 if (bits(s, 1)) {
286 /* get length */
287 symbol = decode(s, &lencode);
288 len = base[symbol] + bits(s, extra[symbol]);
289 if (len == 519) break; /* end code */
290
291 /* get distance */
292 symbol = len == 2 ? 2 : dict;
293 dist = decode(s, &distcode) << symbol;
294 dist += bits(s, symbol);
295 dist++;
296 if (s->first && dist > s->next)
297 return -3; /* distance too far back */
298
299 /* copy length bytes from distance bytes back */
300 do {
301 to = s->out + s->next;
302 from = to - dist;
303 copy = MAXWIN;
304 if (s->next < dist) {
305 from += copy;
306 copy = dist;
307 }
308 copy -= s->next;
309 if (copy > len) copy = len;
310 len -= copy;
311 s->next += copy;
312 do {
313 *to++ = *from++;
314 } while (--copy);
315 } while (len != 0);
316 }
317 else {
318 /* get literal and write it */
319 symbol = lit ? decode(s, &litcode) : bits(s, 8);
320 s->out[s->next++] = symbol;
321 }
322 } while (1);
323 return 0;
324}
325
326static void decompress_subchunk( uint8_t * subchunk, size_t subchunk_size, int chunks, size_t outchunk_size ) {
327 uint8_t output[ outchunk_size ];
328 struct state s; /* input/output state */
329 int err; /* return value */
330
331 memset( output, 0, outchunk_size );
332
333 /* initialize input state */
334 s.in = subchunk;
335 s.left = subchunk_size;
336 while( chunks-- ) {
337 int i;
338
339 /* (Re-)initialize output state */
340 s.out = output;
341 s.next = 0;
342 s.first = 1;
343 s.bitbuf = 0;
344 s.bitcnt = 0;
345
346 err = decomp(&s);
347 if( err ) {
348 for( i=0; i<32; ++i ) fprintf( stderr, "%02X ", s.in[i] );
349 fprintf( stderr, "\nError: %d\n", err );
350 return;
351 }
352
353 /* Dump to stdout for now */
354 fwrite( output, outchunk_size, 1, stdout );
355 }
356}
357
358static void decode_19bit_address( uint8_t const *source, uint32_t *dest, size_t length )
359{
360 uint32_t acc_bits = 0, acc = 0;
361 while( 1 )
362 {
363 acc = acc*256+*(source++); acc_bits+=8;
364 if( acc_bits >= 19 ) {
365 uint32_t tmp = acc >> (acc_bits-19);
366 *(dest++) = (tmp & 0x7ffff) << 11;
367 acc_bits -= 19;
368 if( !length-- ) return;
369 }
370 }
371}
372
373int main( int args, char **argv ) {
374 MAP file;
375 uint32_t offsets[256];
376 uint16_t num_subchunks, subchunk_rest_count, subchunk_one_count;
377 uint8_t *fp, *subchunk;
378 int i;
379
380 if( args < 2 ) {
381 fprintf( stderr, "Syntax: %s FILENAME\n", argv[0] );
382 exit(1);
383 }
384
385 file = map_file( argv[1], 1 );
386 if( !file ) exit( 1 );
387 fp = file->addr;
388
389 num_subchunks = *(uint16_t*)(fp+0x14);
390 subchunk_rest_count = *(uint16_t*)(fp+0x1c);
391 subchunk_one_count = *(uint16_t*)(fp+0x1e);
392 subchunk = fp + 0x20 + ( 19*num_subchunks +7 )/ 8;
393
394 decode_19bit_address ( fp + 0x20, offsets, num_subchunks );
395 offsets[num_subchunks] = file->size;
396
397 decompress_subchunk( subchunk, offsets[i], subchunk_one_count, MAXWIN );
398
399 for( i=0; i< num_subchunks; ++i )
400 if( offsets[i] + 0x800 < file->size )
401 decompress_subchunk( fp + offsets[i] + 0x800, offsets[i+1] - offsets[i], subchunk_rest_count, MAXWIN );
402
403 return 0;
404}
diff --git a/src/export/extract_version_2.c b/src/export/extract_version_2.c
index c4914a7..68b05af 100644
--- a/src/export/extract_version_2.c
+++ b/src/export/extract_version_2.c
@@ -1,130 +1,404 @@
1#include <fcntl.h> 1/*
2#include "mystdlib.h" 2 This decompressor uses modified code from zlib. Below is it's original copyright
3 notice.
4
5 Copyright (C) 2003, 2012 Mark Adler
6 version 1.2, 24 Oct 2012
7
8 This software is provided 'as-is', without any express or implied
9 warranty. In no event will the author be held liable for any damages
10 arising from the use of this software.
11
12 Permission is granted to anyone to use this software for any purpose,
13 including commercial applications, and to alter it and redistribute it
14 freely, subject to the following restrictions:
15
16 1. The origin of this software must not be misrepresented; you must not
17 claim that you wrote the original software. If you use this software
18 in a product, an acknowledgment in the product documentation would be
19 appreciated but is not required.
20 2. Altered source versions must be plainly marked as such, and must not be
21 misrepresented as being the original software.
22 3. This notice may not be removed or altered from any source distribution.
23
24 Mark Adler madler@alumni.caltech.edu
25 */
26
3#include <stdlib.h> 27#include <stdlib.h>
4#include <string.h> 28#include <stdio.h>
5#include <stdint.h> 29#include <stdint.h>
6#include <unistd.h> 30#include "mystdlib.h"
31#include <string.h>
7 32
8/* lha header: 33/*
34uint8_t table[] = {
35 0x00, 0x39, 0x20, 0x15, 0x24, 0x04, 0x61, 0x80, 0xc4, 0xc0, 0x1f, 0xa4, 0x04, 0xca, 0x40, 0xb4,
36 0x08, 0x19, 0xe7, 0x03, 0xab, 0xe0, 0x83, 0xac, 0x12, 0x92, 0x82, 0x95, 0x10, 0x5b, 0x24, 0x0c,
37 0x75, 0x81, 0xaf, 0xe0, 0x3a, 0x24, 0x07, 0xc7, 0xc1, 0x09, 0x88, 0x23, 0x55, 0x84, 0xac, 0x50,
38 0x9d, 0xc6, 0x14, 0xc3, 0x82, 0xb9, 0x30, 0x5b, 0x54, 0x0b, 0xf1, 0x81, 0x8e, 0xbc, 0x33, 0xce,
39 0x86, 0xb9, 0xa0, 0xdf, 0xae, 0x1d, 0x04, 0x03, 0xc0, 0xf8, 0x7c, 0x13, 0x0f, 0xf0, 0x22, 0x0b,
40 0xa8, 0x43, 0x3e, 0x88, 0xa0, 0xd1, 0x1b, 0x48, 0x24, 0x61, 0x84, 0xad, 0xb0, 0x99, 0xe0, 0x13,
41 0xc0, 0x62, 0x87, 0xdc, 0x52, 0xba, 0x8a, 0x8f, 0x31, 0x59, 0x0e, 0x2c, 0x16, 0x85, 0xa0, 0x38,
42 0xb7, 0x8a, 0x17, 0x66, 0xc2, 0xfb, 0x50, 0x61, 0x3f, 0x8c, 0x5f, 0x01, 0x93, 0x76, 0x33, 0x6f,
43 0x46, 0x8e, 0x18, 0xd5, 0x6f, 0x1b, 0x1e, 0x83, 0x71, 0x6c, 0x6f, 0xfc, 0x0e, 0x3d, 0x91, 0xcf,
44 0x08, 0x3a, 0xc4, 0x87, 0x75, 0x80, 0xf2, 0x23, 0x1e, 0xb4, 0x83, 0xe4, 0xe0, 0x7e, 0x6d, 0x10,
45 0x06, 0x22, 0x08, 0x5a, 0x41, 0xf8, 0x48, 0x5b, 0x11, 0x0e, 0xce, 0x22, 0x46, 0x44, 0x56, 0x84,
46 0x8c, 0xc9, 0x91, 0xdc, 0x22, 0x44, 0x26, 0x49, 0x9f, 0x49, 0x56, 0x71, 0x2f, 0x18, 0x26, 0x6d,
47 0xa4, 0xde, 0xdc, 0x9e, 0x02, 0x94, 0x04, 0xc2, 0x89, 0x38, 0x52, 0x31, 0x8a, 0x68, 0x39, 0x51,
48 0x64, 0x2a, 0xb0, 0x45, 0x65, 0x80, 0xae, 0xbf, 0x16, 0x19, 0x12, 0xcb, 0x48, 0x5a, 0x6b, 0xcb,
49 0x6a, 0x11, 0x70, 0xa3, 0x2e, 0x80, 0x85, 0xdd, 0xf0, 0xbd, 0x7a, 0x97, 0xe5, 0xb3, 0x03, 0xbc,
50 0x61, 0x67, 0xcc, 0x49, 0xe1, 0x8d, 0x11, 0x32, 0x28, 0x66, 0x53, 0xd8, 0xcc, 0x2b, 0x19, 0xbd,
51 0x93, 0x3e, 0x54, 0x68, 0xae, 0x4d, 0x35, 0xe1, 0xaa, 0xbe, 0x35, 0xd5, 0x46, 0xc8, 0x30, 0xda,
52 0xd0, 0x9b, 0x95, 0x73, 0x7a, 0xc8, 0x70, 0x43, 0x4e, 0x23, 0x19, 0xc8, 0x41, 0x39, 0x76, 0x07,
53 0x3d, 0xb8, 0xe9, 0x79, 0x9d, 0x65, 0xe3, 0xb3, 0x94, 0x77, 0x47, 0xcf, 0x04, 0xf1, 0xe3, 0xfc,
54 0x3c, 0xf0, 0x27, 0xac, 0x30, 0xf7, 0x4b, 0x1f, 0x25, 0x03, 0xec, 0x0a, 0x7e, 0x65, 0xcf, 0xe8,
55 0x1a, 0x00, 0xa7, 0x40, 0x8d, 0xc8, 0x21, 0x45, 0x05, 0xf5, 0xa0, 0xf7, 0x34, 0x25, 0xd0, 0x85,
56 0x9a, 0x50, 0xcf, 0x02, 0x1d, 0xd2, 0x44, 0x2c, 0x28, 0x93, 0xdd, 0x14, 0x36, 0x22, 0xbf, 0x24,
57 0x5e, 0x86, 0x8c, 0xa9, 0x91, 0xb1, 0x8a, 0x39, 0xd3, 0x47, 0xab, 0x29, 0x03, 0x29, 0x22, 0x28,
58 0xa4, 0x7b, 0xc4, 0x96, 0x68, 0x93, 0xb9, 0x12, 0x92, 0xfa, 0x55, 0xd2, 0x4b, 0x2d, 0x29, 0x74,
59 0x05, 0x30, 0x38, 0xa6, 0x40, 0xd4, 0xcf, 0x6e, 0x9a, 0xcc, 0x13, 0x74, 0xda, 0x72, 0x0a, 0x4e,
60 0xb3, 0x29, 0xe4, 0xa9, 0x3e, 0x60, 0xa8, 0x04, 0xa5, 0x07, 0xce, 0xa1, 0xe7, 0xc0
61};
62*/
9 63
1000 Header length 64#define MAXBITS 13 /* maximum code length */
1101 Header checksum [02-length] 65#define MAXWIN 8192 /* maximum window size */
1202 0x2d ('-')
1303 0x6c ('l')
1404 0x68 ('h')
1505 0x?? ('0' or '5') unsure
1606 0x2d ('-')
1707 0x?? LSB of compressed size
1808 0x?? ..
1909 0x00 ..
2010 0x00 MSB of compressed size, i.e. 0
21..
2221 Length of path name
23 66
67struct state {
68 uint8_t *in; /* next input location */
69 uint8_t *out; /* output buffer and sliding window */
24 70
25*/ 71 unsigned left; /* available input at in */
72 int bitbuf; /* bit buffer */
73 int bitcnt; /* number of bits in bit buffer */
26 74
27static uint8_t mantra_in[] = { 0x68, 0x35, 0x2d, 0x6c }; 75 unsigned next; /* index of next write location in out[] */
76 int first; /* true to check distances (for first 4K) */
77};
28 78
29int main( int args, char **argv ) 79static int bits(struct state *s, int need)
30{ 80{
31 int filenum = 0, run = 1, first_run = 1; 81 int val; /* bit accumulator */
32 size_t offset = 0, old_offset = 0, reported = 0, enc_len = 32; 82
33 uint8_t mantra[4], id0, id5, *mapped_file; 83 /* load at least need bits into val */
34 MAP map; 84 val = s->bitbuf;
35 85 while (s->bitcnt < need) {
36 /* For streets we do have a enc_len of 34 */ 86 val |= (int)(*(s->in)++) << s->bitcnt; /* load eight bits */
37 while( run ) { 87 s->left--;
38 switch( getopt( args, argv, ":e:" ) ) { 88 s->bitcnt += 8;
39 case -1 : run = 0; break;
40 case 'e':
41 enc_len = strtoul( optarg, 0, 0 );
42 break;
43 default:
44 fputs( "Syntax: %s [-e encrypted_length (default: 32, for streets 34 or 0)] path-to-teiln.dat", stderr );
45 exit( 1 );
46 } 89 }
47 }
48 run = 1;
49 90
50 if( optind == args ) 91 /* drop need bits and update buffer, always zero to seven bits left */
51 { fputs( "Missing filename.", stderr ); exit( 1 ); } 92 s->bitbuf = val >> need;
93 s->bitcnt -= need;
52 94
53 map = map_file( argv[optind], 1 ); 95 /* return need bits, zeroing the bits above that */
54 mapped_file = map->addr; 96 return val & ((1 << need) - 1);
97}
55 98
56 mantra[0] = mantra_in[0] ^ mapped_file[4]; 99struct huffman {
57 mantra[1] = mantra_in[1] ^ mapped_file[5]; 100 short *count; /* number of symbols of each length */
58 mantra[2] = mantra_in[2] ^ mapped_file[2]; 101 short *symbol; /* canonically ordered symbols */
59 mantra[3] = mantra_in[3] ^ mapped_file[3]; 102};
60 103
61 id0 = mapped_file[0]; 104static int decode(struct state *s, struct huffman *h)
62 id5 = mapped_file[5]; 105{
106 int len; /* current number of bits in code */
107 int code; /* len bits being decoded */
108 int first; /* first code of length len */
109 int count; /* number of codes of length len */
110 int index; /* index of first code of length len in symbol table */
111 int bitbuf; /* bits from stream */
112 int left; /* bits left in next or left to process */
113 short *next; /* next number of codes */
63 114
64 while( run ) 115 bitbuf = s->bitbuf;
65 { 116 left = s->bitcnt;
66 while( ( offset < map->size ) && ( 117 code = first = index = 0;
67 ( mapped_file[ offset + 0 ] != id0 ) || 118 len = 1;
68 ( mapped_file[ offset + 2 ] != ( '-' ^ mantra[2] )) || 119 next = h->count + 1;
69 ( mapped_file[ offset + 3 ] != ( 'l' ^ mantra[3] )) || 120 while (1) {
70 ( mapped_file[ offset + 4 ] != ( 'h' ^ mantra[0] )) || 121 while (left--) {
71 ( mapped_file[ offset + 5 ] != id5 ) || 122 code |= (bitbuf & 1) ^ 1; /* invert code */
72 ( mapped_file[ offset + 6 ] != ( '-' ^ mantra[2] )) 123 bitbuf >>= 1;
73 ) ) offset++; 124 count = *next++;
74 125 if (code < first + count) { /* if length len, return symbol */
75 // printf( "Found an appropriate offset at: %zd\n", offset ); 126 s->bitbuf = bitbuf;
76 127 s->bitcnt = (s->bitcnt - len) & 7;
77 if( reported < ( offset * 10 ) / map->size ) 128 return h->symbol[index + (code - first)];
78 { 129 }
79 reported++; 130 index += count; /* else update for next length */
80 printf( "%zd%% ", 10 * reported ); 131 first += count;
81 fflush( stdout ); 132 first <<= 1;
133 code <<= 1;
134 len++;
135 }
136 left = (MAXBITS+1) - len;
137 if (left == 0) break;
138 bitbuf = *(s->in)++;
139 s->left--;
140 if (left > 8) left = 8;
141 }
142 return -9; /* ran out of codes */
143}
144
145static int construct(struct huffman *h, const unsigned char *rep, int n)
146{
147 int symbol; /* current symbol when stepping through length[] */
148 int len; /* current length when stepping through h->count[] */
149 int left; /* number of possible codes left of current length */
150 short offs[MAXBITS+1]; /* offsets in symbol table for each length */
151 short length[256]; /* code lengths */
152
153 /* convert compact repeat counts into symbol bit length list */
154 symbol = 0;
155 do {
156 len = *rep++;
157 left = (len >> 4) + 1;
158 len &= 15;
159 do {
160 length[symbol++] = len;
161 } while (--left);
162 } while (--n);
163 n = symbol;
164
165 /* count number of codes of each length */
166 for (len = 0; len <= MAXBITS; len++)
167 h->count[len] = 0;
168 for (symbol = 0; symbol < n; symbol++)
169 (h->count[length[symbol]])++; /* assumes lengths are within bounds */
170 if (h->count[0] == n) /* no codes! */
171 return 0; /* complete, but decode() will fail */
172
173 /* check for an over-subscribed or incomplete set of lengths */
174 left = 1; /* one possible code of zero length */
175 for (len = 1; len <= MAXBITS; len++) {
176 left <<= 1; /* one more bit, double codes left */
177 left -= h->count[len]; /* deduct count from possible codes */
178 if (left < 0) return left; /* over-subscribed--return negative */
179 } /* left > 0 means incomplete */
180
181 /* generate offsets into symbol table for each length for sorting */
182 offs[1] = 0;
183 for (len = 1; len < MAXBITS; len++)
184 offs[len + 1] = offs[len] + h->count[len];
185
186 /*
187 * put symbols in table sorted by length, by symbol order within each
188 * length
189 */
190 for (symbol = 0; symbol < n; symbol++)
191 if (length[symbol] != 0)
192 h->symbol[offs[length[symbol]]++] = symbol;
193
194 /* return zero for complete set, positive for incomplete set */
195 return left;
196}
197
198/*
199 * Decode PKWare Compression Library stream.
200 *
201 * Format notes:
202 *
203 * - First byte is 0 if literals are uncoded or 1 if they are coded. Second
204 * byte is 4, 5, or 6 for the number of extra bits in the distance code.
205 * This is the base-2 logarithm of the dictionary size minus six.
206 *
207 * - Compressed data is a combination of literals and length/distance pairs
208 * terminated by an end code. Literals are either Huffman coded or
209 * uncoded bytes. A length/distance pair is a coded length followed by a
210 * coded distance to represent a string that occurs earlier in the
211 * uncompressed data that occurs again at the current location.
212 *
213 * - A bit preceding a literal or length/distance pair indicates which comes
214 * next, 0 for literals, 1 for length/distance.
215 *
216 * - If literals are uncoded, then the next eight bits are the literal, in the
217 * normal bit order in th stream, i.e. no bit-reversal is needed. Similarly,
218 * no bit reversal is needed for either the length extra bits or the distance
219 * extra bits.
220 *
221 * - Literal bytes are simply written to the output. A length/distance pair is
222 * an instruction to copy previously uncompressed bytes to the output. The
223 * copy is from distance bytes back in the output stream, copying for length
224 * bytes.
225 *
226 * - Distances pointing before the beginning of the output data are not
227 * permitted.
228 *
229 * - Overlapped copies, where the length is greater than the distance, are
230 * allowed and common. For example, a distance of one and a length of 518
231 * simply copies the last byte 518 times. A distance of four and a length of
232 * twelve copies the last four bytes three times. A simple forward copy
233 * ignoring whether the length is greater than the distance or not implements
234 * this correctly.
235 */
236static int decomp(struct state *s)
237{
238 int lit; /* true if literals are coded */
239 int dict; /* log2(dictionary size) - 6 */
240 int symbol; /* decoded symbol, extra bits for distance */
241 int len; /* length for copy */
242 unsigned dist; /* distance for copy */
243 int copy; /* copy counter */
244 unsigned char *from, *to; /* copy pointers */
245 static int virgin = 1; /* build tables once */
246 static short litcnt[MAXBITS+1], litsym[256]; /* litcode memory */
247 static short lencnt[MAXBITS+1], lensym[16]; /* lencode memory */
248 static short distcnt[MAXBITS+1], distsym[64]; /* distcode memory */
249 static struct huffman litcode = {litcnt, litsym}; /* length code */
250 static struct huffman lencode = {lencnt, lensym}; /* length code */
251 static struct huffman distcode = {distcnt, distsym};/* distance code */
252 /* bit lengths of literal codes */
253 static const unsigned char litlen[] = {
254 11, 124, 8, 7, 28, 7, 188, 13, 76, 4, 10, 8, 12, 10, 12, 10, 8, 23, 8,
255 9, 7, 6, 7, 8, 7, 6, 55, 8, 23, 24, 12, 11, 7, 9, 11, 12, 6, 7, 22, 5,
256 7, 24, 6, 11, 9, 6, 7, 22, 7, 11, 38, 7, 9, 8, 25, 11, 8, 11, 9, 12,
257 8, 12, 5, 38, 5, 38, 5, 11, 7, 5, 6, 21, 6, 10, 53, 8, 7, 24, 10, 27,
258 44, 253, 253, 253, 252, 252, 252, 13, 12, 45, 12, 45, 12, 61, 12, 45,
259 44, 173};
260 /* bit lengths of length codes 0..15 */
261 static const unsigned char lenlen[] = {2, 35, 36, 53, 38, 23};
262 /* bit lengths of distance codes 0..63 */
263 static const unsigned char distlen[] = {2, 20, 53, 230, 247, 151, 248};
264 static const short base[16] = { /* base for length codes */
265 3, 2, 4, 5, 6, 7, 8, 9, 10, 12, 16, 24, 40, 72, 136, 264};
266 static const char extra[16] = { /* extra bits for length codes */
267 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8};
268
269 /* set up decoding tables (once--might not be thread-safe) */
270 if (virgin) {
271 construct(&litcode, litlen, sizeof(litlen));
272 construct(&lencode, lenlen, sizeof(lenlen));
273 construct(&distcode, distlen, sizeof(distlen));
274 virgin = 0;
82 } 275 }
83 276
84 if( offset == map->size ) 277 /* read header */
85 run = 0; 278 lit = bits(s, 8);
86 279 if (lit > 1) return -1;
87 if( !first_run ) 280 dict = bits(s, 8);
88 { 281 if (dict < 4 || dict > 6) return -2;
89 uint8_t *mf = mapped_file + old_offset, df[128]; 282
90 size_t filename_len, header_len, i; 283 /* decode literals and length/distance pairs */
91 int fh; 284 do {
92 char filename[32]; 285 if (bits(s, 1)) {
93 286 /* get length */
94 /* De-"crypt" obfuscation to our header copy */ 287 symbol = decode(s, &lencode);
95 for( i=0; i<enc_len; ++i) 288 len = base[symbol] + bits(s, extra[symbol]);
96 df[i] = mf[i] ^ mantra[i%4]; 289 if (len == 519) break; /* end code */
97 290
98 /* Get values from LHA header */ 291 /* get distance */
99 header_len = df[0] + 2; 292 symbol = len == 2 ? 2 : dict;
100 filename_len = df[21]; 293 dist = decode(s, &distcode) << symbol;
101 294 dist += bits(s, symbol);
102 /* Copy rest of header, so we can checksum */ 295 dist++;
103 for( i=enc_len; i<header_len; ++i) 296 if (s->first && dist > s->next)
104 df[i] = mf[i]; 297 return -3; /* distance too far back */
105 298
106 /* Make up new sequental file name */ 299 /* copy length bytes from distance bytes back */
107 snprintf( filename, sizeof(filename), "%0*d.lha", (int)filename_len, filenum++ ); 300 do {
108 memcpy( ((uint8_t*)df) + 22, filename, filename_len); 301 to = s->out + s->next;
109 302 from = to - dist;
110 /* Recalculate checksum with new file name */ 303 copy = MAXWIN;
111 df[1] = 0; for( i=2; i<header_len; ++i) df[1] += df[i]; 304 if (s->next < dist) {
112 305 from += copy;
113 /* Open file and dump our de-"crypted" header and then rest of file */ 306 copy = dist;
114 fh = open( filename, O_CREAT | O_TRUNC | O_WRONLY, 0644 ); 307 }
115 if( enc_len > header_len ) { 308 copy -= s->next;
116 write( fh, df, enc_len ); 309 if (copy > len) copy = len;
117 write( fh, mf + enc_len, offset - old_offset - enc_len ); 310 len -= copy;
118 } else { 311 s->next += copy;
119 write( fh, df, header_len ); 312 do {
120 write( fh, mf + header_len, offset - old_offset - header_len ); 313 *to++ = *from++;
314 } while (--copy);
315 } while (len != 0);
121 } 316 }
122 close( fh ); 317 else {
318 /* get literal and write it */
319 symbol = lit ? decode(s, &litcode) : bits(s, 8);
320 s->out[s->next++] = symbol;
321 }
322 } while (1);
323 return 0;
324}
325
326static void decompress_subchunk( uint8_t * subchunk, size_t subchunk_size, int chunks, size_t outchunk_size ) {
327 uint8_t output[ outchunk_size ];
328 struct state s; /* input/output state */
329 int err; /* return value */
330
331 memset( output, 0, outchunk_size );
332
333 /* initialize input state */
334 s.in = subchunk;
335 s.left = subchunk_size;
336 while( chunks-- ) {
337 int i;
338
339 /* (Re-)initialize output state */
340 s.out = output;
341 s.next = 0;
342 s.first = 1;
343 s.bitbuf = 0;
344 s.bitcnt = 0;
345
346 err = decomp(&s);
347 if( err ) {
348 for( i=0; i<32; ++i ) fprintf( stderr, "%02X ", s.in[i] );
349 fprintf( stderr, "\nError: %d\n", err );
350 return;
123 } 351 }
124 first_run = 0; 352
125 old_offset = offset++; 353 /* Dump to stdout for now */
354 fwrite( output, outchunk_size, 1, stdout );
126 } 355 }
356}
357
358static void decode_19bit_address( uint8_t const *source, uint32_t *dest, size_t length )
359{
360 uint32_t acc_bits = 0, acc = 0;
361 while( 1 )
362 {
363 acc = acc*256+*(source++); acc_bits+=8;
364 if( acc_bits >= 19 ) {
365 uint32_t tmp = acc >> (acc_bits-19);
366 *(dest++) = (tmp & 0x7ffff) << 11;
367 acc_bits -= 19;
368 if( !length-- ) return;
369 }
370 }
371}
372
373int main( int args, char **argv ) {
374 MAP file;
375 uint32_t offsets[256];
376 uint16_t num_subchunks, subchunk_rest_count, subchunk_one_count;
377 uint8_t *fp, *subchunk;
378 int i;
379
380 if( args < 2 ) {
381 fprintf( stderr, "Syntax: %s FILENAME\n", argv[0] );
382 exit(1);
383 }
384
385 file = map_file( argv[1], 1 );
386 if( !file ) exit( 1 );
387 fp = file->addr;
388
389 num_subchunks = *(uint16_t*)(fp+0x14);
390 subchunk_rest_count = *(uint16_t*)(fp+0x1c);
391 subchunk_one_count = *(uint16_t*)(fp+0x1e);
392 subchunk = fp + 0x20 + ( 19*num_subchunks +7 )/ 8;
393
394 decode_19bit_address ( fp + 0x20, offsets, num_subchunks );
395 offsets[num_subchunks] = file->size;
396
397 decompress_subchunk( subchunk, offsets[i], subchunk_one_count, MAXWIN );
398
399 for( i=0; i< num_subchunks; ++i )
400 if( offsets[i] + 0x800 < file->size )
401 decompress_subchunk( fp + offsets[i] + 0x800, offsets[i+1] - offsets[i], subchunk_rest_count, MAXWIN );
127 402
128 unmap_file( &map );
129 return 0; 403 return 0;
130} 404}
diff --git a/src/export/extract_version_3.c b/src/export/extract_version_3.c
index e0e858d..c4914a7 100644
--- a/src/export/extract_version_3.c
+++ b/src/export/extract_version_3.c
@@ -1,70 +1,130 @@
1#include <stdlib.h>
2#include <stdio.h>
3#include <unistd.h>
4#include <fcntl.h> 1#include <fcntl.h>
5#include <string.h>
6#include <zlib.h>
7#include "mystdlib.h" 2#include "mystdlib.h"
3#include <stdlib.h>
4#include <string.h>
5#include <stdint.h>
6#include <unistd.h>
8 7
9#define XORLEN (29) 8/* lha header:
10#define HUGEBLOCK (8*1024*1024)
11
12int main(int argc, char **argv) {
13 unsigned const char xorkey [XORLEN] = "Just for Fun. Linus Torvalds.";
14 unsigned char input [XORLEN];
15 unsigned char output [HUGEBLOCK];
16 char respath[32]; /* file_XXXXX\0 */
17 int zres = 0, filenum = 0, resfile;
18 size_t i, offs = 0, reported = 0;
19 MAP in;
20
21 if( argc != 2 ) exit(111);
22 in = map_file( argv[1], 1 );
23
24 z_stream z; memset( &z, 0, sizeof(z));
25
26 while( offs < in->size ) {
27 size_t inlen = offs + XORLEN < in->size ? XORLEN : in->size - offs;
28 for( i=0; i<inlen; ++i ) input[i] = in->addr[offs+i] ^ xorkey[i];
29 z.next_in = input; z.avail_in = inlen;
30 z.next_out = output; z.avail_out = HUGEBLOCK;
31 inflateInit( &z ); zres = inflate( &z, Z_NO_FLUSH );
32 if( (zres != Z_OK) && (zres != Z_STREAM_END) )
33 goto error_continue;
34
35 z.next_in = in->addr + offs + inlen;
36 z.avail_in = (unsigned int)(in->size - offs - inlen);
37 while( zres == Z_OK ) zres = inflate( &z, Z_NO_FLUSH );
38
39 if( zres != Z_STREAM_END ) {
40error_continue:
41 inflateEnd(&z); memset( &z, 0, sizeof(z));
42 offs++;
43 continue;
44 }
45 9
46 sprintf( respath, "file_%05X", filenum++ ); 1000 Header length
1101 Header checksum [02-length]
1202 0x2d ('-')
1303 0x6c ('l')
1404 0x68 ('h')
1505 0x?? ('0' or '5') unsure
1606 0x2d ('-')
1707 0x?? LSB of compressed size
1808 0x?? ..
1909 0x00 ..
2010 0x00 MSB of compressed size, i.e. 0
21..
2221 Length of path name
47 23
48 resfile = open( respath, O_RDWR | O_CREAT, 0644 ); 24
49 if( resfile < 0 ) { 25*/
50 fprintf( stderr, "Could not open output file %s\n", respath ); 26
51 exit(1); 27static uint8_t mantra_in[] = { 0x68, 0x35, 0x2d, 0x6c };
28
29int main( int args, char **argv )
30{
31 int filenum = 0, run = 1, first_run = 1;
32 size_t offset = 0, old_offset = 0, reported = 0, enc_len = 32;
33 uint8_t mantra[4], id0, id5, *mapped_file;
34 MAP map;
35
36 /* For streets we do have a enc_len of 34 */
37 while( run ) {
38 switch( getopt( args, argv, ":e:" ) ) {
39 case -1 : run = 0; break;
40 case 'e':
41 enc_len = strtoul( optarg, 0, 0 );
42 break;
43 default:
44 fputs( "Syntax: %s [-e encrypted_length (default: 32, for streets 34 or 0)] path-to-teiln.dat", stderr );
45 exit( 1 );
52 } 46 }
53 write( resfile, output, z.total_out ); 47 }
54 close( resfile ); 48 run = 1;
55 offs += z.total_in; 49
50 if( optind == args )
51 { fputs( "Missing filename.", stderr ); exit( 1 ); }
52
53 map = map_file( argv[optind], 1 );
54 mapped_file = map->addr;
56 55
57 if( reported < ( offs * 10 ) / in->size ) { 56 mantra[0] = mantra_in[0] ^ mapped_file[4];
57 mantra[1] = mantra_in[1] ^ mapped_file[5];
58 mantra[2] = mantra_in[2] ^ mapped_file[2];
59 mantra[3] = mantra_in[3] ^ mapped_file[3];
60
61 id0 = mapped_file[0];
62 id5 = mapped_file[5];
63
64 while( run )
65 {
66 while( ( offset < map->size ) && (
67 ( mapped_file[ offset + 0 ] != id0 ) ||
68 ( mapped_file[ offset + 2 ] != ( '-' ^ mantra[2] )) ||
69 ( mapped_file[ offset + 3 ] != ( 'l' ^ mantra[3] )) ||
70 ( mapped_file[ offset + 4 ] != ( 'h' ^ mantra[0] )) ||
71 ( mapped_file[ offset + 5 ] != id5 ) ||
72 ( mapped_file[ offset + 6 ] != ( '-' ^ mantra[2] ))
73 ) ) offset++;
74
75 // printf( "Found an appropriate offset at: %zd\n", offset );
76
77 if( reported < ( offset * 10 ) / map->size )
78 {
58 reported++; 79 reported++;
59 printf( "%zd%% ", 10 * reported ); 80 printf( "%zd%% ", 10 * reported );
60 fflush( stdout ); 81 fflush( stdout );
61 } 82 }
62 83
63 inflateEnd(&z); memset( &z, 0, sizeof(z)); 84 if( offset == map->size )
85 run = 0;
86
87 if( !first_run )
88 {
89 uint8_t *mf = mapped_file + old_offset, df[128];
90 size_t filename_len, header_len, i;
91 int fh;
92 char filename[32];
93
94 /* De-"crypt" obfuscation to our header copy */
95 for( i=0; i<enc_len; ++i)
96 df[i] = mf[i] ^ mantra[i%4];
97
98 /* Get values from LHA header */
99 header_len = df[0] + 2;
100 filename_len = df[21];
101
102 /* Copy rest of header, so we can checksum */
103 for( i=enc_len; i<header_len; ++i)
104 df[i] = mf[i];
105
106 /* Make up new sequental file name */
107 snprintf( filename, sizeof(filename), "%0*d.lha", (int)filename_len, filenum++ );
108 memcpy( ((uint8_t*)df) + 22, filename, filename_len);
109
110 /* Recalculate checksum with new file name */
111 df[1] = 0; for( i=2; i<header_len; ++i) df[1] += df[i];
112
113 /* Open file and dump our de-"crypted" header and then rest of file */
114 fh = open( filename, O_CREAT | O_TRUNC | O_WRONLY, 0644 );
115 if( enc_len > header_len ) {
116 write( fh, df, enc_len );
117 write( fh, mf + enc_len, offset - old_offset - enc_len );
118 } else {
119 write( fh, df, header_len );
120 write( fh, mf + header_len, offset - old_offset - header_len );
121 }
122 close( fh );
123 }
124 first_run = 0;
125 old_offset = offset++;
64 } 126 }
65 unmap_file(&in); 127
66 if( reported < 10 ) 128 unmap_file( &map );
67 printf( "100%% " );
68 fflush( stdout );
69 return 0; 129 return 0;
70} 130}
diff --git a/src/export/extract_version_4.c b/src/export/extract_version_4.c
new file mode 100644
index 0000000..e0e858d
--- /dev/null
+++ b/src/export/extract_version_4.c
@@ -0,0 +1,70 @@
1#include <stdlib.h>
2#include <stdio.h>
3#include <unistd.h>
4#include <fcntl.h>
5#include <string.h>
6#include <zlib.h>
7#include "mystdlib.h"
8
9#define XORLEN (29)
10#define HUGEBLOCK (8*1024*1024)
11
12int main(int argc, char **argv) {
13 unsigned const char xorkey [XORLEN] = "Just for Fun. Linus Torvalds.";
14 unsigned char input [XORLEN];
15 unsigned char output [HUGEBLOCK];
16 char respath[32]; /* file_XXXXX\0 */
17 int zres = 0, filenum = 0, resfile;
18 size_t i, offs = 0, reported = 0;
19 MAP in;
20
21 if( argc != 2 ) exit(111);
22 in = map_file( argv[1], 1 );
23
24 z_stream z; memset( &z, 0, sizeof(z));
25
26 while( offs < in->size ) {
27 size_t inlen = offs + XORLEN < in->size ? XORLEN : in->size - offs;
28 for( i=0; i<inlen; ++i ) input[i] = in->addr[offs+i] ^ xorkey[i];
29 z.next_in = input; z.avail_in = inlen;
30 z.next_out = output; z.avail_out = HUGEBLOCK;
31 inflateInit( &z ); zres = inflate( &z, Z_NO_FLUSH );
32 if( (zres != Z_OK) && (zres != Z_STREAM_END) )
33 goto error_continue;
34
35 z.next_in = in->addr + offs + inlen;
36 z.avail_in = (unsigned int)(in->size - offs - inlen);
37 while( zres == Z_OK ) zres = inflate( &z, Z_NO_FLUSH );
38
39 if( zres != Z_STREAM_END ) {
40error_continue:
41 inflateEnd(&z); memset( &z, 0, sizeof(z));
42 offs++;
43 continue;
44 }
45
46 sprintf( respath, "file_%05X", filenum++ );
47
48 resfile = open( respath, O_RDWR | O_CREAT, 0644 );
49 if( resfile < 0 ) {
50 fprintf( stderr, "Could not open output file %s\n", respath );
51 exit(1);
52 }
53 write( resfile, output, z.total_out );
54 close( resfile );
55 offs += z.total_in;
56
57 if( reported < ( offs * 10 ) / in->size ) {
58 reported++;
59 printf( "%zd%% ", 10 * reported );
60 fflush( stdout );
61 }
62
63 inflateEnd(&z); memset( &z, 0, sizeof(z));
64 }
65 unmap_file(&in);
66 if( reported < 10 )
67 printf( "100%% " );
68 fflush( stdout );
69 return 0;
70}
diff --git a/src/export/split_version_2.c b/src/export/split_version_3.c
index 2019762..2019762 100644
--- a/src/export/split_version_2.c
+++ b/src/export/split_version_3.c