From b4bf8417af0d8ebff2c50570c70fdecaf6a53ed9 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Wed, 20 Mar 2019 04:30:29 +0100 Subject: Add code to lookup new zip codes for 1995 entries and fix up streetnames --- src/postprocess/postprocess-1992.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/postprocess/postprocess-1992.sh (limited to 'src/postprocess/postprocess-1992.sh') diff --git a/src/postprocess/postprocess-1992.sh b/src/postprocess/postprocess-1992.sh new file mode 100644 index 0000000..1e685d2 --- /dev/null +++ b/src/postprocess/postprocess-1992.sh @@ -0,0 +1,32 @@ +# Generate file with all relevant columns from 1992 +paste 1992_Q2/{01_Flags,12_Vorwahl,12_Vorwahl_block,11_Ort,07_Strasse,08_Hausnummer,02_Nachname,03_Vorname,13_Rufnummer,10_Postleitzahl_West,10_Zustellamt_PLZOst} > 1992-fvvoshnvrpp.txt + +# Generate lookup file from 1995 +paste 1995_Q0/{12_Vorwahl,11_Ort,07_Strasse,08_Hausnummer,02_Nachname,03_Vorname,13_Rufnummer,10_Postleitzahl} | tr '\n\t' '\0' > 1995-voshnvrp.bin + +# To debug in lldb +process launch -i 1992_testfile.txt -- 1995-vorwahl-ort-strasse-hnr-name-vorname-rufnummer-plz.bin + +# Compile plz mapper +cc -O3 -o map_plz map_plz.c -I ../src/export/ ../src/export/mystdlib.c + +# outputs mapped plz, generates brutemap.txt +touch brutemap_input.bin zip_simple_map.bin +./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 10_Postleitzahl + +# generate street name translation table from brutemap, +# only taking into account similar street names +# cut -f 3,4 brutemap.txt | tr '[:upper:]' '[:lower:]' | paste brutemap.txt - | cut -f 1-4,6,7 | ./jaro | cut -f 1-5 > brutemap_filtered.txt + +# generate street name translation table from brutemap, +# only taking into account similar street names, new style +cut -f 3,4 brutemap.txt | python simi.py | paste - brutemap.txt > brutemap_simifiltered.txt + +# Sort and prepare similarity filtered files for the merge +cut -f 1-5 brutemap_simifiltered.txt | sort | uniq -c | sed -E $'s:^ *([[:digit:]]+) :\\1\t:' | tr '\n\t' '\0' > brutemap_input.bin + +# compile zipmap into a binary format +sort -u zip_mapfile.txt | tr '\n' '\0' > zip_simple_map.bin + +# Redo the mapping with the data from brutemap and zipmap +./map_plz 1995-voshnvrp.bin zip_simple_map.bin brutemap_input.bin < 1992-fvvoshnvrpp.txt > 10_Postleitzahl -- cgit v1.2.3