Merge branch 'aRkedos-unique-file'

6 files changed, 384 insertions, 33 deletions

diff --git a/doc/ranger.1 b/doc/ranger.1
index e3896877..5324ccc3 100644
--- a/doc/ranger.1
+++ b/doc/ranger.1
@@ -133,7 +133,7 @@
 .\" ========================================================================
 .\"
 .IX Title "RANGER 1"
-.TH RANGER 1 "ranger-1.9.2" "2019-12-28" "ranger manual"
+.TH RANGER 1 "ranger-1.9.2" "2019-12-30" "ranger manual"
 .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
 .\" way too many mistakes in technical documents.
 .if n .ad l
@@ -778,12 +778,6 @@ Close the current tab.  The last tab cannot be closed this way.
 A key chain that allows you to quickly change the line mode of all the files of
 the current directory.  For a more permanent solution, use the command
 \&\*(L"default_linemode\*(R" in your rc.conf.
-.IP ".n" 14
-.IX Item ".n"
-Apply a new filename filter.
-.IP ".m" 14
-.IX Item ".m"
-Apply a new mimetype filter.
 .IP ".d" 14
 .IX Item ".d"
 Apply the typefilter \*(L"directory\*(R".
@@ -793,19 +787,32 @@ Apply the typefilter \*(L"file\*(R".
 .IP ".l" 14
 .IX Item ".l"
 Apply the typefilter \*(L"symlink\*(R".
+.IP ".m" 14
+.IX Item ".m"
+Apply a new mimetype filter.
+.IP ".n" 14
+.IX Item ".n"
+Apply a new filename filter.
+.IP ".#" 14
+Apply a new hash filter.
+.IP ".""" 14
+Apply a new duplicate filter.
+.IP ".'" 14
+Apply a new unique filter.
 .IP ".|" 14
-Combine the two topmost filters from the filter stack in the \*(L"\s-1OR\*(R"\s0
-relationship, instead of the \*(L"\s-1AND\*(R"\s0 used implicitly.
+Combine the two topmost filters from the filter stack in the \f(CW\*(C`OR\*(C'\fR
+relationship, instead of the \f(CW\*(C`AND\*(C'\fR used implicitly.
 .IP ".&" 14
-Explicitly combine the two topmost filters in the \*(L"\s-1AND\*(R"\s0
-relationship. Usually not needed though might be useful in more
-complicated scenarios.
+Explicitly combine the two topmost filters in the \f(CW\*(C`AND\*(C'\fR relationship.
+Usually not needed because filters are implicitly in this relationship though
+might be useful in more complicated scenarios.
 .IP ".!" 14
 Negate the topmost filter.
 .IP ".r" 14
 .IX Item ".r"
-Rotate the filter stack by N elements. Just confirm with enter to
-rotate by 1, i.e. move the topmost element to the bottom of the stack.
+Rotate the filter stack by N elements. Where N is provided as a numeric prefix
+like vim's \fIcount\fR and defaults to 1, i.e. move the topmost element to the
+bottom of the stack.
 .IP ".c" 14
 .IX Item ".c"
 Clear the filter stack.
@@ -1391,6 +1398,91 @@ Displays only the files of specified inode type. To display only directories,
 use the 'd' parameter. To display only files, use the 'f' parameter. To display
 only links, use the 'l' parameter. Parameters can be combined. To remove this
 filter, use no parameter.
+.IP "filter_stack [\fIcommand\fR [\fIargs\fR]]" 2
+.IX Item "filter_stack [command [args]]"
+Manage the filter stack, adding, removing and manipulating filters. For
+example, to show only duplicate files and symlinks:
+.Sp
+.Vb 5
+\&  :filter_stack add type f
+\&  :filter_stack add duplicate
+\&  :filter_stack add and
+\&  :filter_stack add type l
+\&  :filter_stack add or
+.Ve
+.Sp
+Or using the mapped keys:
+.Sp
+.Vb 1
+\&  .f ." .& .l .|
+.Ve
+.Sp
+Available subcommands:
+.RS 2
+.IP "add \s-1FILTER_TYPE\s0 [\s-1ARGS...\s0]" 2
+.IX Item "add FILTER_TYPE [ARGS...]"
+Add a new filter to the top of the filter stack. Each filter on the stack is
+applied in turn, resulting in an implicit logical \f(CW\*(C`AND\*(C'\fR relation. The
+following \f(CW\*(C`FILTER_TYPE\*(C'\fRs are available:
+.RS 2
+.IP "duplicate" 2
+.IX Item "duplicate"
+Filter files so only files that have duplicates in the same directory are
+shown. Useful when cleaning up identical songs and memes that were saved using
+distinct file names.
+.IP "filename \s-1NAME\s0" 2
+.IX Item "filename NAME"
+Filter files that contain \s-1NAME\s0 in the filename, regular expression syntax is
+allowed.
+.IP "hash \s-1PATH\s0" 2
+.IX Item "hash PATH"
+Filter files so only files with the same hash as \s-1PATH\s0 are shown.
+.IP "mimetype \s-1TYPE\s0" 2
+.IX Item "mimetype TYPE"
+Filter files of a certain \s-1MIME\s0 type, regular expression syntax is allowed.
+.IP "typefilter [d|f|l]" 2
+.IX Item "typefilter [d|f|l]"
+Filter files of a certain type, \f(CW\*(C`d\*(C'\fR for directories, \f(CW\*(C`f\*(C'\fR for files and \f(CW\*(C`l\*(C'\fR
+for symlinks.
+.IP "unique" 2
+.IX Item "unique"
+Filter files so only unique files and the oldest file of every set of
+duplicates is shown.
+.IP "and" 2
+.IX Item "and"
+Explicitly combine the two topmost filters in the \*(L"\s-1AND\*(R"\s0 relationship.
+Usually not needed because filters are implicitly in this relationship though
+might be useful in more complicated scenarios.
+.IP "not" 2
+.IX Item "not"
+Negate the topmost filter.
+.IP "or" 2
+.IX Item "or"
+Combine the two topmost filters from the filter stack in the \*(L"\s-1OR\*(R"\s0
+relationship, instead of the \*(L"\s-1AND\*(R"\s0 used implicitly.
+.RE
+.RS 2
+.RE
+.IP "pop" 2
+.IX Item "pop"
+Pop the topmost filter from the filter stack.
+.IP "decompose" 2
+.IX Item "decompose"
+Decompose the topmost filter combinator (e.g. \f(CW\*(C`.!\*(C'\fR, \f(CW\*(C`.|\*(C'\fR).
+.IP "rotate [N=1]" 2
+.IX Item "rotate [N=1]"
+Rotate the filter stack by N elements. Where N is passed as argument or as a
+numeric prefix like vim's \fIcount\fR, default to 1, i.e. move the topmost element
+to the bottom of the stack.
+.IP "clear" 2
+.IX Item "clear"
+Clear the filter stack.
+.IP "show" 2
+.IX Item "show"
+Show the current filter stack state.
+.RE
+.RS 2
+.RE
 .IP "find \fIpattern\fR" 2
 .IX Item "find pattern"
 Search files in the current directory that contain the given (case-insensitive)
@@ -1799,6 +1891,15 @@ Specifies the number of spaces to use to replace tabs in \fIhighlight\fRed files
 \&\fIhighlight\fR will pick up command line options specified in this variable. A
 \&\f(CW\*(C`\-\-style=\*(C'\fR option specified here will override \f(CW\*(C`HIGHLIGHT_STYLE\*(C'\fR. Similarly,
 \&\f(CW\*(C`\-\-replace\-tabs=\*(C'\fR will override \f(CW\*(C`HIGHLIGHT_TABWIDTH\*(C'\fR.
+.IP "\s-1OPENSCAD_COLORSCHEME\s0" 8
+.IX Item "OPENSCAD_COLORSCHEME"
+Specifies the colorscheme used by \fIopenscad\fR while previewing 3D models. Read
+\&\fIopenscad\fR man page for colorschemes. Ranger will default to Tomorrow Night.
+.IP "\s-1OPENSCAD_IMGSIZE\s0" 8
+.IX Item "OPENSCAD_IMGSIZE"
+Specifies the internal resolution \fIopenscad\fR will use for rendering 3D models.
+The image will be downscaled to fit the preview pane. This resolution will
+default to \*(L"1000,1000\*(R" if no value is set.
 .IP "\s-1XDG_CONFIG_HOME\s0" 8
 .IX Item "XDG_CONFIG_HOME"
 Specifies the directory for configuration files. Defaults to \fI\f(CI$HOME\fI/.config\fR.
diff --git a/doc/ranger.pod b/doc/ranger.pod
index 33b16b92..bae332e9 100644
--- a/doc/ranger.pod
+++ b/doc/ranger.pod
@@ -772,14 +772,6 @@ A key chain that allows you to quickly change the line mode of all the files of
 the current directory.  For a more permanent solution, use the command
 "default_linemode" in your rc.conf.
 
-=item .n
-
-Apply a new filename filter.
-
-=item .m
-
-Apply a new mimetype filter.
-
 =item .d
 
 Apply the typefilter "directory".
@@ -792,16 +784,36 @@ Apply the typefilter "file".
 
 Apply the typefilter "symlink".
 
+=item .m
+
+Apply a new mimetype filter.
+
+=item .n
+
+Apply a new filename filter.
+
+=item .#
+
+Apply a new hash filter.
+
+=item ."
+
+Apply a new duplicate filter.
+
+=item .'
+
+Apply a new unique filter.
+
 =item .|
 
-Combine the two topmost filters from the filter stack in the "OR"
-relationship, instead of the "AND" used implicitly.
+Combine the two topmost filters from the filter stack in the C<OR>
+relationship, instead of the C<AND> used implicitly.
 
 =item .&
 
-Explicitly combine the two topmost filters in the "AND"
-relationship. Usually not needed though might be useful in more
-complicated scenarios.
+Explicitly combine the two topmost filters in the C<AND> relationship.
+Usually not needed because filters are implicitly in this relationship though
+might be useful in more complicated scenarios.
 
 =item .!
 
@@ -809,8 +821,9 @@ Negate the topmost filter.
 
 =item .r
 
-Rotate the filter stack by N elements. Just confirm with enter to
-rotate by 1, i.e. move the topmost element to the bottom of the stack.
+Rotate the filter stack by N elements. Where N is provided as a numeric prefix
+like vim's I<count> and defaults to 1, i.e. move the topmost element to the
+bottom of the stack.
 
 =item .c
 
@@ -1500,6 +1513,103 @@ use the 'd' parameter. To display only files, use the 'f' parameter. To display
 only links, use the 'l' parameter. Parameters can be combined. To remove this
 filter, use no parameter.
 
+=item filter_stack [I<command> [I<args>]]
+
+Manage the filter stack, adding, removing and manipulating filters. For
+example, to show only duplicate files and symlinks:
+
+  :filter_stack add type f
+  :filter_stack add duplicate
+  :filter_stack add and
+  :filter_stack add type l
+  :filter_stack add or
+
+Or using the mapped keys:
+
+  .f ." .& .l .|
+
+Available subcommands:
+
+=over 2
+
+=item add FILTER_TYPE [ARGS...]
+
+Add a new filter to the top of the filter stack. Each filter on the stack is
+applied in turn, resulting in an implicit logical C<AND> relation. The
+following C<FILTER_TYPE>s are available:
+
+=over 2
+
+=item duplicate
+
+Filter files so only files that have duplicates in the same directory are
+shown. Useful when cleaning up identical songs and memes that were saved using
+distinct file names.
+
+=item filename NAME
+
+Filter files that contain NAME in the filename, regular expression syntax is
+allowed.
+
+=item hash PATH
+
+Filter files so only files with the same hash as PATH are shown.
+
+=item mimetype TYPE
+
+Filter files of a certain MIME type, regular expression syntax is allowed.
+
+=item typefilter [d|f|l]
+
+Filter files of a certain type, C<d> for directories, C<f> for files and C<l>
+for symlinks.
+
+=item unique
+
+Filter files so only unique files and the oldest file of every set of
+duplicates is shown.
+
+=item and
+
+Explicitly combine the two topmost filters in the "AND" relationship.
+Usually not needed because filters are implicitly in this relationship though
+might be useful in more complicated scenarios.
+
+=item not
+
+Negate the topmost filter.
+
+=item or
+
+Combine the two topmost filters from the filter stack in the "OR"
+relationship, instead of the "AND" used implicitly.
+
+=back
+
+=item pop
+
+Pop the topmost filter from the filter stack.
+
+=item decompose
+
+Decompose the topmost filter combinator (e.g. C<.!>, C<.|>).
+
+=item rotate [N=1]
+
+Rotate the filter stack by N elements. Where N is passed as argument or as a
+numeric prefix like vim's I<count>, default to 1, i.e. move the topmost element
+to the bottom of the stack.
+
+=item clear
+
+Clear the filter stack.
+
+=item show
+
+Show the current filter stack state.
+
+=back
+
 =item find I<pattern>
 
 Search files in the current directory that contain the given (case-insensitive)
diff --git a/ranger/config/commands.py b/ranger/config/commands.py
index e66ce849..5defa677 100755
--- a/ranger/config/commands.py
+++ b/ranger/config/commands.py
@@ -1691,7 +1691,7 @@ class filter_stack(Command):
         elif subcommand == "clear":
             self.fm.thisdir.filter_stack = []
         elif subcommand == "rotate":
-            rotate_by = int(self.arg(2) or 1)
+            rotate_by = int(self.arg(2) or self.quantifier or 1)
             self.fm.thisdir.filter_stack = (
                 self.fm.thisdir.filter_stack[-rotate_by:]
                 + self.fm.thisdir.filter_stack[:-rotate_by]
diff --git a/ranger/config/rc.conf b/ranger/config/rc.conf
index 7226130d..66a5fbbc 100644
--- a/ranger/config/rc.conf
+++ b/ranger/config/rc.conf
@@ -596,15 +596,18 @@ map zf    console filter%space
 copymap zf zz
 
 # Filter stack
-map .n console filter_stack add name%space
-map .m console filter_stack add mime%space
 map .d filter_stack add type d
 map .f filter_stack add type f
 map .l filter_stack add type l
+map .m console filter_stack add mime%space
+map .n console filter_stack add name%space
+map .# console filter_stack add hash%space
+map ." filter_stack add duplicate
+map .' filter_stack add unique
 map .| filter_stack add or
 map .& filter_stack add and
 map .! filter_stack add not
-map .r console filter_stack rotate
+map .r filter_stack rotate
 map .c filter_stack clear
 map .* filter_stack decompose
 map .p filter_stack pop
diff --git a/ranger/core/filter_stack.py b/ranger/core/filter_stack.py
index 2ca2b1c5..59495437 100644
--- a/ranger/core/filter_stack.py
+++ b/ranger/core/filter_stack.py
@@ -7,8 +7,17 @@ from __future__ import (absolute_import, division, print_function)
 
 import re
 import mimetypes
+# pylint: disable=invalid-name
+try:
+    from itertools import izip_longest as zip_longest
+except ImportError:
+    from itertools import zip_longest
+# pylint: enable=invalid-name
+from os.path import abspath
 
 from ranger.container.directory import accept_file, InodeFilterConstants
+from ranger.core.shared import FileManagerAware
+from ranger.ext.hash import hash_chunks
 
 # pylint: disable=too-few-public-methods
 
@@ -65,6 +74,104 @@ class MimeFilter(BaseFilter):
         return "<Filter: mimetype =~ /{}/>".format(self.pattern)
 
 
+@stack_filter("hash")
+class HashFilter(BaseFilter, FileManagerAware):
+    def __init__(self, filepath = None):
+        if filepath is None:
+            self.filepath = self.fm.thisfile.path
+        else:
+            self.filepath = filepath
+        if self.filepath is None:
+            self.fm.notify("Error: No file selected for hashing!", bad=True)
+        # TODO: Lazily generated list would be more efficient, a generator
+        #       isn't enough because this object is reused for every fsobject
+        #       in the current directory.
+        self.filehash = list(hash_chunks(abspath(self.filepath)))
+
+    def __call__(self, fobj):
+        for (chunk1, chunk2) in zip_longest(self.filehash,
+                                            hash_chunks(fobj.path),
+                                            fillvalue=''):
+            if chunk1 != chunk2:
+                return False
+        return True
+
+    def __str__(self):
+        return "<Filter: hash {}>".format(self.filepath)
+
+
+def group_by_hash(fsobjects):
+    hashes = {}
+    for fobj in fsobjects:
+        chunks = hash_chunks(fobj.path)
+        chunk = next(chunks)
+        while chunk in hashes:
+            for dup in hashes[chunk]:
+                _, dup_chunks = dup
+                try:
+                    hashes[next(dup_chunks)] = [dup]
+                    hashes[chunk].remove(dup)
+                except StopIteration:
+                    pass
+            try:
+                chunk = next(chunks)
+            except StopIteration:
+                hashes[chunk].append((fobj, chunks))
+                break
+        else:
+            hashes[chunk] = [(fobj, chunks)]
+
+    groups = []
+    for dups in hashes.values():
+        group = []
+        for (dup, _) in dups:
+            group.append(dup)
+        if group:
+            groups.append(group)
+
+    return groups
+
+
+@stack_filter("duplicate")
+class DuplicateFilter(BaseFilter, FileManagerAware):
+    def __init__(self, _):
+        self.duplicates = self.get_duplicates()
+
+    def __call__(self, fobj):
+        return fobj in self.duplicates
+
+    def __str__(self):
+        return "<Filter: duplicate>"
+
+    def get_duplicates(self):
+        duplicates = set()
+        for dups in group_by_hash(self.fm.thisdir.files_all):
+            if len(dups) >= 2:
+                duplicates.update(dups)
+        return duplicates
+
+
+@stack_filter("unique")
+class UniqueFilter(BaseFilter, FileManagerAware):
+    def __init__(self, _):
+        self.unique = self.get_unique()
+
+    def __call__(self, fobj):
+        return fobj in self.unique
+
+    def __str__(self):
+        return "<Filter: unique>"
+
+    def get_unique(self):
+        unique = set()
+        for dups in group_by_hash(self.fm.thisdir.files_all):
+            try:
+                unique.add(min(dups, key=lambda fobj: fobj.stat.st_ctime))
+            except ValueError:
+                pass
+        return unique
+
+
 @stack_filter("type")
 class TypeFilter(BaseFilter):
     type_to_function = {
diff --git a/ranger/ext/hash.py b/ranger/ext/hash.py
new file mode 100644
index 00000000..d9b2234b
--- /dev/null
+++ b/ranger/ext/hash.py
@@ -0,0 +1,30 @@
+# This file is part of ranger, the console file manager.
+# License: GNU GPL version 3, see the file "AUTHORS" for details.
+
+from __future__ import (absolute_import, division, print_function)
+
+from os import listdir
+from os.path import getsize, isdir
+from hashlib import sha256
+
+# pylint: disable=invalid-name
+
+
+def hash_chunks(filepath, h=None):
+    if not h:
+        h = sha256()
+    if isdir(filepath):
+        h.update(filepath)
+        yield h.hexdigest()
+        for fp in listdir(filepath):
+            for fp_chunk in hash_chunks(fp, h=h):
+                yield fp_chunk
+    elif getsize(filepath) == 0:
+        yield h.hexdigest()
+    else:
+        with open(filepath, 'rb') as f:
+            # Read the file in ~64KiB chunks (multiple of sha256's block
+            # size that works well enough with HDDs and SSDs)
+            for chunk in iter(lambda: f.read(h.block_size * 1024), b''):
+                h.update(chunk)
+                yield h.hexdigest()