From 54d31710a4976163d74e4040cd6c2ad8c055c43d Mon Sep 17 00:00:00 2001 From: hukl Date: Tue, 31 Mar 2009 22:56:23 +0200 Subject: renamed and refactored even further --- lib/chaos_importer.rb | 336 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 lib/chaos_importer.rb (limited to 'lib/chaos_importer.rb') diff --git a/lib/chaos_importer.rb b/lib/chaos_importer.rb new file mode 100644 index 00000000..3a307a35 --- /dev/null +++ b/lib/chaos_importer.rb @@ -0,0 +1,336 @@ +require 'iconv' +require 'nokogiri' +require 'lib/chaos_calendar/ical_occurrences' +require 'digest/sha1' + + +class ChaosXml + attr_reader :xml, :unique_name, :locale, :date, :slug + + def initialize options = {} + options.each_pair do |key,value| + instance_variable_set "@#{key}", value + instance_eval("def #{key}; @#{key};end") + end + end +end + +class ChaosImporter + include Enumerable + + def initialize path + unless Node.root + Node.create! + end + + @directory = path + @years = {} + end + + # Iterates through all xml files within the @directory and yields ChaosXml + # objects with the parsed attributes. These attributes are the basic data + # needed for further processing / parsing. + def each + directories = Dir.glob("#{@directory}/*/*.xml{,.de,.en}") + + directories.each do |path| + next if path =~ /index\.xml/ + + chaos_id = chaos_id_from_path( path ) + options = {} + options[:xml] = Nokogiri::XML( File.new(path).read ) + options[:locale] = lang_from_path( path ) + options[:date] = options[:xml].at("//date").content.to_date + options[:slug] = chaos_id + options[:unique_name] = "updates/#{options[:date]}/#{options[:slug]}" + xml = ChaosXml.new options + + yield xml + end + end + + # Uses the each method to loop over the xml files and uses the attrubutes of + # the returned ChaosXml objects to do some further processing which is needed + # to create proper ActiveRecord records + def import_updates + unless @updates = Node.find_by_unique_name('updates') + @updates = Node.create!( :slug => 'updates' ) + @updates.move_to_child_of Node.root + end + + self.each do |update| + author = find_or_create_author( update ) + node = find_or_create_node( update ) + html = convert_to_html( update.xml ) + page = fill_draft_with_content(node.draft, html, update.locale) + + add_tags_to_page page, update.xml, "update" + add_event_to_node node, update.xml if page.tag_list.include?("event") + page.user = author + page.save + puts node.unique_name + end + + Node.all.each {|node| node.publish_draft!} + end + + def lang_from_path path + case path + when /\.de$/ then :de + when /\.en$/ then :en + else + :de + end + end + + def chaos_id_from_path path + path.sub(@directory, "").split(/\//).last.split(/\./)[0] + end + + def find_or_create_author update + login = update.xml.at("//author").content rescue "webmaster" + puts login + + # password = Digest::SHA1.hexdigest("#{Time.now+rand(100).days}") + # unless author = User.find_by_login(login) + # author = User.create!( + # :login => login, + # :email => "#{login}@example.com", + # :password => password, + # :password_confirmation => password + # ) + # end + + # author + + + end + + def find_or_create_node update + year = update.date.year + + unique_name_array = update.unique_name.split("/") + + unless @years[year] || (@years[year] = Node.find_by_unique_name("updates/#{year}")) + @years[year] = Node.create :slug => year + @years[year].move_to_child_of @updates + end + + unless node = Node.find_by_unique_name(update.unique_name) + node = Node.create :slug => update.slug + node.move_to_child_of @years[year] + end + + node + end + + def fill_draft_with_content draft, html, lang + I18n.locale = lang + + options = { + :title => html.xpath("//title")[0].content, + :abstract => html.xpath("//abstract")[0].content, + :body => extract_body(html) + } + + draft.update_attributes options + draft + end + + def extract_body html, excluded_tags=[] + default_excluded_tags = [ + "DTSTART", + "DTEND", + "DURATION", + "LOCATION", + "GEO", + "SUMMARY", + "URL" + ] + + excluded_tags = (default_excluded_tags + excluded_tags).uniq + + body = "" + element = html.xpath("//abstract")[0].next_sibling + + while element do + body << element.to_s unless excluded_tags.include? element.name + element = element.next_sibling + end + + body + end + + def add_tags_to_page page, chaospage, *custom_tags + tag_list = custom_tags + + chaospage.xpath("//flags").each do |node| + node.each do |k,v| + case k + when "calendar" + tag_list << "event" + when "pm" + tag_list << "pressemitteilung" + end + end + end + + # Getting rid of duplicate flags + tag_list.uniq! + + page.tag_list = tag_list.join(",") + page.save + end + + def add_event_to_node node, chaospage + rrule = get_rrule(chaospage) + + event_options = { + :start_time => get_start_time(chaospage), + :end_time => get_end_time(chaospage), + :allday => is_allday?(chaospage), + :rrule => rrule, + :custom_rrule => is_custom_rrule?(rrule), + :location => get_location(chaospage), + :url => get_url(chaospage), + :latitude => get_latitude(chaospage), + :longitude => get_logitude(chaospage) + } + + unless tmp_event = node.event + tmp_event = Event.create! event_options.merge({:node_id => node.id}) + else + tmp_event.update_attributes event_options + end + end + + def get_start_time chaospage + chaospage.at("//ical:DTSTART").content || raise("DTSTART not found") + end + + def get_end_time chaospage + dtstart = chaospage.at("//ical:DTSTART") + dtend = chaospage.at("//ical:DTEND") + duration = chaospage.at("//ical:DURATION") + + if dtend + return dtend.content + elsif duration + parsed_duration = Ical_occurrences.duration_to_fixnum(duration.content) + return (dtstart.content.to_time + parsed_duration) + else + raise("Neiter DTEND nor DURATION found") + end + end + + def is_allday? chaospage + !chaospage.at("//ical:DTSTART").[]("VALUE").nil? + end + + def get_rrule chaospage + if rrule = chaospage.at("//ical:RRULE") + rrtxt = '' + rrule.children.each do |subrule| + rule_name = subrule.name + rule_content = subrule.content.sub(/\W/,'') + + next if rule_content.blank? + + rrtxt += "#{rule_name}=#{rule_content};" + end + rrtxt.chomp!(';') + rrtxt + else + nil + end + end + + def is_custom_rrule? rrule + default_rules = [ + "FREQ=WEEKLY;INTERVAL=1", + "FREQ=MONTHLY;INTERVAL=1", + "FREQ=YEARLY;INTERVAL=1" + ] + + rrule && !default_rules.include?(rrule) ? true : false + end + + def get_location chaospage + location = chaospage.at("//ical:LOCATION") + location ? location.content : nil + end + + def get_url chaospage + location = chaospage.at("//ical:LOCATION") + location.[]("ALTREP") if location + end + + def get_latitude chaospage + geo = chaospage.at("//ical:GEO") + geo.text.split(";")[0] if geo + end + + def get_logitude chaospage + geo = chaospage.at("//ical:GEO") + geo.text.split(";")[1] if geo + end + + def convert_to_html chaospage + + chaospage.xpath('//paragraph').each {|sub| sub.name = "p"} + chaospage.xpath('//quote').each {|sub| sub.name = "blockquote" } + chaospage.xpath('//subtitle').each {|sub| sub.name = "h3" } + chaospage.xpath('//strong').each {|sub| sub.name = "em" } + chaospage.xpath('//stronger').each {|sub| sub.name = "strong" } + chaospage.xpath('//chapter').each {|sub| sub.name = "h2" } + + chaospage.xpath('//link').each do |sub| + sub.name = "a" + href = sub.[]("ref") + sub.remove_attribute("ref") + sub.[]=("href", href) + sub.remove_attribute("type") + end + + chaospage.xpath('//list').each do |sub| + if !sub.css("row item").empty? + sub.name = "table" + + sub.css("row").each {|x| x.name = "tr"} + sub.css("tr item").each {|x| x.name = "td"} + elsif !sub.css("item").empty? + sub.name = "ul" + + sub.css("item").each {|x| x.name = "li"} + end + end + + chaospage.xpath('//media').each do |sub| + sub.name = "img" + src = sub.[]("ref") + sub.remove_attribute("src") + sub.[]=("src", src) + unless sub.content + sub.[]=("alt", sub.content) + sub.xpath('//*').each {|x| x.remove} + end + end + + chaospage.xpath('//name').each do |sub| + if sub.[]("email") + mail_href = "mailto:#{sub.[]('email')}" + sub.remove_attribute("email") + sub.[]=("href", mail_href) + end + sub.name = "a" + + if href = sub.[]("ref") + sub.remove_attribute("ref") + sub.[]=("href", href) + end + end + + chaospage + + end +end \ No newline at end of file -- cgit v1.3 From ebbad4011bc5652966bd25d8455da208c7f9ab7f Mon Sep 17 00:00:00 2001 From: hukl Date: Tue, 31 Mar 2009 23:46:02 +0200 Subject: mini fixes --- lib/chaos_importer.rb | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'lib/chaos_importer.rb') diff --git a/lib/chaos_importer.rb b/lib/chaos_importer.rb index 3a307a35..657f1f04 100644 --- a/lib/chaos_importer.rb +++ b/lib/chaos_importer.rb @@ -42,7 +42,7 @@ class ChaosImporter options[:locale] = lang_from_path( path ) options[:date] = options[:xml].at("//date").content.to_date options[:slug] = chaos_id - options[:unique_name] = "updates/#{options[:date]}/#{options[:slug]}" + options[:unique_name] = "updates/#{options[:date].year}/#{options[:slug]}" xml = ChaosXml.new options yield xml @@ -68,7 +68,7 @@ class ChaosImporter add_event_to_node node, update.xml if page.tag_list.include?("event") page.user = author page.save - puts node.unique_name + # puts node.unique_name end Node.all.each {|node| node.publish_draft!} @@ -88,8 +88,8 @@ class ChaosImporter end def find_or_create_author update - login = update.xml.at("//author").content rescue "webmaster" - puts login + # login = update.xml.at("//author").content rescue "webmaster" + # puts login # password = Digest::SHA1.hexdigest("#{Time.now+rand(100).days}") # unless author = User.find_by_login(login) @@ -101,13 +101,12 @@ class ChaosImporter # ) # end - # author - - + # author end def find_or_create_node update year = update.date.year + unique_name_array = update.unique_name.split("/") @@ -116,6 +115,8 @@ class ChaosImporter @years[year].move_to_child_of @updates end + puts update.unique_name + unless node = Node.find_by_unique_name(update.unique_name) node = Node.create :slug => update.slug node.move_to_child_of @years[year] @@ -127,12 +128,18 @@ class ChaosImporter def fill_draft_with_content draft, html, lang I18n.locale = lang + draft.reload + options = { :title => html.xpath("//title")[0].content, :abstract => html.xpath("//abstract")[0].content, :body => extract_body(html) } + if draft.node.slug == "wahlcomputer-hessen" + puts "#{I18n.locale} >>> #{lang} >>> #{options[:title]}" + end + draft.update_attributes options draft end @@ -278,7 +285,7 @@ class ChaosImporter def convert_to_html chaospage chaospage.xpath('//paragraph').each {|sub| sub.name = "p"} - chaospage.xpath('//quote').each {|sub| sub.name = "blockquote" } + chaospage.xpath('//quote').each {|sub| sub.name = "em" } chaospage.xpath('//subtitle').each {|sub| sub.name = "h3" } chaospage.xpath('//strong').each {|sub| sub.name = "em" } chaospage.xpath('//stronger').each {|sub| sub.name = "strong" } -- cgit v1.3 From 41ee3ef53f85f0529b2dd6e44f0b4fbd0f3777a2 Mon Sep 17 00:00:00 2001 From: hukl Date: Tue, 31 Mar 2009 23:46:11 +0200 Subject: don't even look --- lib/chaos_importer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/chaos_importer.rb') diff --git a/lib/chaos_importer.rb b/lib/chaos_importer.rb index 657f1f04..f8e98f52 100644 --- a/lib/chaos_importer.rb +++ b/lib/chaos_importer.rb @@ -59,6 +59,7 @@ class ChaosImporter end self.each do |update| + author = find_or_create_author( update ) node = find_or_create_node( update ) html = convert_to_html( update.xml ) @@ -68,7 +69,8 @@ class ChaosImporter add_event_to_node node, update.xml if page.tag_list.include?("event") page.user = author page.save - # puts node.unique_name + + puts node.unique_name end Node.all.each {|node| node.publish_draft!} @@ -115,8 +117,6 @@ class ChaosImporter @years[year].move_to_child_of @updates end - puts update.unique_name - unless node = Node.find_by_unique_name(update.unique_name) node = Node.create :slug => update.slug node.move_to_child_of @years[year] -- cgit v1.3 From 39671a357fcc410687ceeeb27ae51e0a926cd5ec Mon Sep 17 00:00:00 2001 From: hukl Date: Tue, 7 Apr 2009 21:54:49 +0200 Subject: adding the new chaos_importer.rb to the setup_environment task. removed libxml-ruby dependency --- lib/chaos_importer.rb | 25 ++++++++----------------- lib/tasks/development_init.rake | 31 ++++++++++--------------------- 2 files changed, 18 insertions(+), 38 deletions(-) (limited to 'lib/chaos_importer.rb') diff --git a/lib/chaos_importer.rb b/lib/chaos_importer.rb index f8e98f52..74e0bbed 100644 --- a/lib/chaos_importer.rb +++ b/lib/chaos_importer.rb @@ -73,7 +73,9 @@ class ChaosImporter puts node.unique_name end + puts ">> Publishing Drafts" Node.all.each {|node| node.publish_draft!} + puts ">> Finished" end def lang_from_path path @@ -90,20 +92,13 @@ class ChaosImporter end def find_or_create_author update - # login = update.xml.at("//author").content rescue "webmaster" - # puts login - - # password = Digest::SHA1.hexdigest("#{Time.now+rand(100).days}") - # unless author = User.find_by_login(login) - # author = User.create!( - # :login => login, - # :email => "#{login}@example.com", - # :password => password, - # :password_confirmation => password - # ) - # end + login = update.xml.at("//author").content rescue "webmaster" + + unless author = User.find_by_login(login.downcase) + author = User.find_by_login("webmaster") + end - # author + author end def find_or_create_node update @@ -136,10 +131,6 @@ class ChaosImporter :body => extract_body(html) } - if draft.node.slug == "wahlcomputer-hessen" - puts "#{I18n.locale} >>> #{lang} >>> #{options[:title]}" - end - draft.update_attributes options draft end diff --git a/lib/tasks/development_init.rake b/lib/tasks/development_init.rake index 789cdf6a..00ef83ac 100644 --- a/lib/tasks/development_init.rake +++ b/lib/tasks/development_init.rake @@ -3,7 +3,12 @@ require 'csv' namespace :cccms do desc "Setup everythin" - task :setup_environment => [:create_admin_user, :import_updates, :create_home_page] do |t| + task :setup_environment => [ + :create_admin_user, + :import_authors, + :import_updates, + :create_home_page + ] do |t| end @@ -19,30 +24,14 @@ namespace :cccms do desc "Import the authors" task :import_authors => :environment do |t| - I18n.locale = :en - @parsed_file = CSV::Reader.parse(File.open("#{RAILS_ROOT}/db/authors.csv")) - - @parsed_file.each_with_index do |row, index| - next if row[0].nil? - - unless author = User.find_by_login(row[0]) - puts "#{row[0]} >> #{row[2]}" - author = User.create!( - :login => row[0], - #:realname => row[1], - :email => row[2], - :password => "foobartrallala", - :password_confirmation => "foobartrallala" - ) - end - - end + importer = AuthorsImporter.new("#{RAILS_ROOT}/db/authors.csv") + importer.import_authors end desc "Import the old XML Files" task :import_updates => :environment do |t| - i = UpdateImporter.new("#{RAILS_ROOT}/db/updates") - i.import_xml + i = ChaosImporter.new("#{RAILS_ROOT}/db/updates") + i.import_updates end desc "Create Home Page" -- cgit v1.3