Use chardet in case of encoding detection problems

2 files changed, 20 insertions, 5 deletions

diff --git a/README b/README
index 5221574c..bd960d47 100644
--- a/README
+++ b/README
@@ -61,6 +61,7 @@ Dependencies
 Optional:
 * The "file" program
 * A pager ("less" by default)
+* The python module "chardet", in case of encoding detection problems
 
 For scope.sh: (enhanced file previews)
 * img2txt (from caca-utils) for previewing images
diff --git a/ranger/core/loader.py b/ranger/core/loader.py
index 4341595c..7fd7dbab 100644
--- a/ranger/core/loader.py
+++ b/ranger/core/loader.py
@@ -23,7 +23,11 @@ import math
 import os
 import sys
 import select
-
+try:
+	import chardet
+	HAVE_CHARDET = True
+except:
+	HAVE_CHARDET = False
 
 class Loadable(object):
 	paused = False
@@ -89,13 +93,13 @@ class CommandLoader(Loadable, SignalDispatcher, FileManagerAware):
 						if rd == process.stderr:
 							read = rd.readline()
 							if py3:
-								read = read.decode('utf-8')
+								read = safeDecode(read)
 							if read:
 								self.fm.notify(read, bad=True)
 						elif rd == process.stdout:
 							read = rd.read(512)
 							if py3:
-								read = read.decode('utf-8')
+								read = safeDecode(read)
 							if read:
 								self.stdout_buffer += read
 				except select.error:
@@ -103,12 +107,12 @@ class CommandLoader(Loadable, SignalDispatcher, FileManagerAware):
 			if not self.silent:
 				for l in process.stderr.readlines():
 					if py3:
-						l = l.decode('utf-8')
+						l = safeDecode(l)
 					self.fm.notify(l, bad=True)
 			if self.read:
 				read = process.stdout.read()
 				if py3:
-					read = read.decode('utf-8')
+					read = safeDecode(read)
 				self.stdout_buffer += read
 		self.finished = True
 		self.signal_emit('after', process=process, loader=self)
@@ -137,6 +141,16 @@ class CommandLoader(Loadable, SignalDispatcher, FileManagerAware):
 			self.process.kill()
 
 
+def safeDecode(string):
+	try:
+		return string.decode("utf-8")
+	except (UnicodeDecodeError):
+		if HAVE_CHARDET:
+			return string.decode(chardet.detect(str)["encoding"])
+		else:
+			return ""
+
+
 class Loader(FileManagerAware):
 	seconds_of_work_time = 0.03
 	throbber_chars = r'/-\|'