Set file id to nil when encounter UTF-8 issues

hulet · Sep 16, 2016 · bdf611b · bdf611b
1 parent 94acdd6
commit bdf611b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,5 @@ tmp
 
 ## RUBINIUS
 *.rbc
+
+test.rb
diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb
@@ -78,10 +78,9 @@ def get_file_list_curated
     file_list_curated = Hash.new
     [index_file_list_raw, all_file_list_raw].each do |file|
       file.each_line do |line|
-        next if line.size < 20
         file_timestamp = line[0..13].to_i
         file_url = line[15..-2]
-        file_id = file_url.split('/')[3..-1].join('/')
+        file_id = file_url.split('/')[3..-1].join('/') rescue nil
         file_id = CGI::unescape file_id
         file_id = file_id.tidy_bytes unless file_id == ""
         if file_id.nil?