From 959576ad96a5a7a53cb9b9f6c5e5fbb6ecff1bda Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Wed, 27 May 2020 01:14:11 +0200 Subject: Remove circular dependency on makefile --- bin/cleaner.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 bin/cleaner.py (limited to 'bin/cleaner.py') diff --git a/bin/cleaner.py b/bin/cleaner.py new file mode 100755 index 00000000..d8c999da --- /dev/null +++ b/bin/cleaner.py @@ -0,0 +1,62 @@ +#! venv/bin/python +import psycopg2 +import psycopg2.extras +import pathlib +import os.path +import datetime +import pypandoc +from bs4 import BeautifulSoup +from datetime import timedelta, datetime, tzinfo +#from langdetect import detect +import sys +import re + +conn = psycopg2.connect(database="cccms_dev", user="postgres", password="", host="127.0.0.1") +cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + +cursor.execute("select * from page_translations") +page_translations = cursor.fetchall() + +fo = open("orig.txt", "w") +fm = open("modi.txt", "w") + +for translation in page_translations: + body = translation.get('body') + cursor.execute("select * from pages where id = %s", [translation['page_id']]) + page = cursor.fetchone() + + if not body: + continue + + if not '