# Copyright (C) 2009, 2010 Roman Zimbelmann <romanz@lavabit.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
if __name__ == '__main__': from __init__ import init; init()
import unittest
from collections import deque
from ranger.ext.iter_tools import *
class TestCases(unittest.TestCase):
def test_flatten(self):
def f(x):
return list(flatten(x))
self.assertEqual(
[1,2,3,4,5],
f([1,2,3,4,5]))
self.assertEqual(
[1,2,3,4,5],
f([1,[2,3],4,5]))
self.assertEqual(
[1,2,3,4,5],
f([[1,[2,3]],4,5]))
self.assertEqual(
[],
f([[[[]]]]))
self.assertEqual(
['a', 'b', 'fskldfjl'],
f(['a', ('b', 'fskldfjl')]))
self.assertEqual(
['a', 'b', 'fskldfjl'],
f(['a', deque(['b', 'fskldfjl'])]))
self.assertEqual(
set([3.5, 4.3, 5.2, 6.0]),
set(f([6.0, set((3.5, 4.3)), (5.2, )])))
def test_unique(self):
def u(x):
return list(unique(x))
self.assertEqual(
[1,2,3],
u([1,2,3]))
self.assertEqual(
[1,2,3],
upre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long *//* HTVMS_WAISProt.h
*
* Adaptation for Lynx by F.Macrides (macrides@sci.wfeb.edu)
*
* 31-May-1994 FM Initial version.
*
*----------------------------------------------------------------------*/
/*
* Routines originally from WProt.h -- FM
*
*----------------------------------------------------------------------*/
/* WIDE AREA INFORMATION SERVER SOFTWARE:
* No guarantees or restrictions. See the readme file for the full standard
* disclaimer.
*
* 3.26.90 Harry Morris, morris@think.com
* 3.30.90 Harry Morris
* - removed chunk code from WAISSearchAPDU,
* - added makeWAISQueryType1Query() and readWAISType1Query() which
* replace makeWAISQueryTerms() and makeWAISQueryDocs().
* 4.11.90 HWM - added definitions of wais element set names
* 4.14.90 HWM - changed symbol for relevance feedback query from QT_3 to
* QT_RelevanceFeedbackQuery added QT_TextRetrievalQuery as a
* synonym for QT_BooleanQuery
* - renamed makeWAISType1Query() to makeWAISTextQuery()
* renamed readWAISType1Query() to readWAISTextQuery()
* 5.29.90 TS - added CSTFreeWAISFoo functions
*/
#ifndef _H_WAIS_protocol_
#define _H_WAIS_protocol_
#ifndef HTUTILS_H
#include <HTUtils.h>
#endif
#include <HTVMS_WaisUI.h>
/*----------------------------------------------------------------------*/
/* Data types / constants */
/* date factor constants */
#define DF_INDEPENDENT 1
#define DF_LATER 2
#define DF_EARLIER 3
#define DF_SPECIFIED_RANGE 4
/* chunk types */
#define CT_document 0
#define CT_byte 1
#define CT_line 2
#define CT_paragraph 3
/* relevance feedback query */
#define QT_RelevanceFeedbackQuery "3"
#define QT_TextRetrievalQuery QT_BooleanQuery
/* new data tags */
#define DT_UserInformationLength (data_tag)99
#define DT_ChunkCode (data_tag)100
#define DT_ChunkIDLength (data_tag)101
#define DT_ChunkMarker (data_tag)102
#define DT_HighlightMarker (data_tag)103
#define DT_DeHighlightMarker (data_tag)104
#define DT_NewlineCharacters (data_tag)105
#define DT_SeedWords (data_tag)106
#define DT_DocumentIDChunk (data_tag)107
#define DT_ChunkStartID (data_tag)108
#define DT_ChunkEndID (data_tag)109
#define DT_TextList (data_tag)110
#define DT_DateFactor (data_tag)111
#define DT_BeginDateRange (data_tag)112
#define DT_EndDateRange (data_tag)113
#define DT_MaxDocumentsRetrieved (data_tag)114
#define DT_SeedWordsUsed (data_tag)115
#define DT_DocumentID (data_tag)116
#define DT_VersionNumber (data_tag)117
#define DT_Score (data_tag)118
#define DT_BestMatch (data_tag)119
#define DT_DocumentLength (data_tag)120
#define DT_Source (data_tag)121
#define DT_Date (data_tag)122
#define DT_Headline (data_tag)123
#define DT_OriginCity (data_tag)124
#define DT_PresentStartByte (data_tag)125
#define DT_TextLength (data_tag)126
#define DT_DocumentText (data_tag)127
#define DT_StockCodes (data_tag)128
#define DT_CompanyCodes (data_tag)129
#define DT_IndustryCodes (data_tag)130
/* added by harry */
#define DT_DocumentHeaderGroup (data_tag)150
#define DT_DocumentShortHeaderGroup (data_tag)151
#define DT_DocumentLongHeaderGroup (data_tag)152
#define DT_DocumentTextGroup (data_tag)153
#define DT_DocumentHeadlineGroup (data_tag)154
#define DT_DocumentCodeGroup (data_tag)155
#define DT_Lines (data_tag)131
#define DT_TYPE_BLOCK (data_tag)132
#define DT_TYPE (data_tag)133
/* wais element sets */
#define ES_DocumentHeader "Document Header"
#define ES_DocumentShortHeader "Document Short Header"
#define ES_DocumentLongHeader "Document Long Header"
#define ES_DocumentText "Document Text"
#define ES_DocumentHeadline "Document Headline"
#define ES_DocumentCodes "Document Codes"
typedef struct DocObj { /* specifies a section of a document */
any *DocumentID;
char *Type;
long ChunkCode;
union {
long Pos;
any *ID;
} ChunkStart;
union {
long Pos;
any *ID;
} ChunkEnd;
} DocObj;
/*----------------------------------------------------------------------*/
/* WAIS APDU extensions */
typedef struct WAISInitResponse {
long ChunkCode;
long ChunkIDLength;
char *ChunkMarker;
char *HighlightMarker;
char *DeHighlightMarker;
char *NewlineCharacters;
/* XXX need to add UpdateFrequency and Update Time */
} WAISInitResponse;
typedef struct WAISSearch {
char *SeedWords;
DocObj **Docs;
char **TextList;
long DateFactor;
char *BeginDateRange;
char *EndDateRange;
long MaxDocumentsRetrieved;
} WAISSearch;
typedef struct WAISDocumentHeader {
any *DocumentID;
long VersionNumber;
long Score;
long BestMatch;
long DocumentLength;
long Lines;
char **Types;
char *Source;
char *Date;
char *Headline;
char *OriginCity;
} WAISDocumentHeader;
typedef struct WAISDocumentShortHeader {
any *DocumentID;
long VersionNumber;
long Score;
long BestMatch;
long DocumentLength;
long Lines;
} WAISDocumentShortHeader;
typedef struct WAISDocumentLongHeader {
any *DocumentID;
long VersionNumber;
long Score;
long BestMatch;
long DocumentLength;
long Lines;
char **Types;
char *Source;
char *Date;
char *Headline;
char *OriginCity;
char *StockCodes;
char *CompanyCodes;
char *IndustryCodes;
} WAISDocumentLongHeader;
typedef struct WAISDocumentText {
any *DocumentID;
long VersionNumber;
any *DocumentText;
} WAISDocumentText;
typedef struct WAISDocumentHeadlines {
any *DocumentID;
long VersionNumber;
char *Source;
char *Date;
char *Headline;
char *OriginCity;
} WAISDocumentHeadlines;
typedef struct WAISDocumentCodes {
any *DocumentID;
long VersionNumber;
char *StockCodes;
char *CompanyCodes;
char *IndustryCodes;
} WAISDocumentCodes;
typedef struct WAISSearchResponse {
char *SeedWordsUsed;
WAISDocumentHeader **DocHeaders;
WAISDocumentShortHeader **ShortHeaders;
WAISDocumentLongHeader **LongHeaders;
WAISDocumentText **Text;
WAISDocumentHeadlines **Headlines;
WAISDocumentCodes **Codes;
diagnosticRecord **Diagnostics;
} WAISSearchResponse;
/*----------------------------------------------------------------------*/
/* Functions */
char *generate_search_apdu(char *buff, /* buffer to hold the apdu */
long *buff_len, /* number of bytes written to the buffer */
char *seed_words, /* string of the seed words */
char *database_name,
DocObj **docobjs,
long maxDocsRetrieved);
DocObj *makeDocObjUsingWholeDocument(any *aDocID, char *type);
DocObj *makeDocObjUsingBytes(any *aDocID, char *type, long start, long end);
DocObj *makeDocObjUsingLines(any *aDocID, char *type, long start, long end);
DocObj *makeDocObjUsingParagraphs(any *aDocID, char *type, any *start, any *end);
void freeDocObj(DocObj *doc);
WAISInitResponse *makeWAISInitResponse(long chunkCode, long chunkIDLen,
char *chunkMarker, char *highlightMarker,
char *deHighlightMarker, char *newLineChars);
void freeWAISInitResponse(WAISInitResponse *init);
WAISSearch *makeWAISSearch(char *seedWords,
DocObj **docs,
char **textList,
long dateFactor,
char *beginDateRange,
char *endDateRange,
long maxDocsRetrieved);
void freeWAISSearch(WAISSearch *query);
WAISDocumentHeader *makeWAISDocumentHeader(any *aDocID,
long versionNumber,
long score,
long bestMatch,
long docLen,
long lines,
char **types,
char *source,
char *date,
char *headline,
char *originCity);
void freeWAISDocumentHeader(WAISDocumentHeader *header);
char *writeWAISDocumentHeader(WAISDocumentHeader *header, char *buffer, long *len);
char *readWAISDocumentHeader(WAISDocumentHeader **header, char *buffer);
WAISDocumentShortHeader *makeWAISDocumentShortHeader(any *aDocID,
long versionNumber,
long score,
long bestMatch,
long docLen,
long lines);
void freeWAISDocumentShortHeader(WAISDocumentShortHeader *header);
char *writeWAISDocumentShortHeader(WAISDocumentShortHeader *header,
char *buffer, long *len);
char *readWAISDocumentShortHeader(WAISDocumentShortHeader **header, char *buffer);
WAISDocumentLongHeader *makeWAISDocumentLongHeader(any *aDocID,
long versionNumber,
long score,
long bestMatch,
long docLen,
long lines,
char **types,
char *source,
char *date, char *headline,
char *originCity,
char *stockCodes,
char *companyCodes,
char *industryCodes);
void freeWAISDocumentLongHeader(WAISDocumentLongHeader *header);
char *writeWAISDocumentLongHeader(WAISDocumentLongHeader *header,
char *buffer,
long *len);
char *readWAISDocumentLongHeader(WAISDocumentLongHeader **header, char *buffer);
WAISSearchResponse *makeWAISSearchResponse(char *seedWordsUsed,
WAISDocumentHeader **docHeaders,
WAISDocumentShortHeader **shortHeaders,
WAISDocumentLongHeader **longHeaders,
WAISDocumentText **text, WAISDocumentHeadlines **headlines,
WAISDocumentCodes **codes,
diagnosticRecord ** diagnostics);
void freeWAISSearchResponse(WAISSearchResponse * response);
WAISDocumentText *makeWAISDocumentText(any *aDocID, long versionNumber,
any *documentText);
void freeWAISDocumentText(WAISDocumentText *docText);
char *writeWAISDocumentText(WAISDocumentText *docText, char *buffer, long *len);
char *readWAISDocumentText(WAISDocumentText **docText, char *buffer);
WAISDocumentHeadlines *makeWAISDocumentHeadlines(any *aDocID,
long versionNumber,
char *source,
char *date,
char *headline,
char *originCity);
void freeWAISDocumentHeadlines(WAISDocumentHeadlines *docHeadline);
char *writeWAISDocumentHeadlines(WAISDocumentHeadlines *docHeadline,
char *buffer,
long *len);
char *readWAISDocumentHeadlines(WAISDocumentHeadlines **docHeadline, char *buffer);
WAISDocumentCodes *makeWAISDocumentCodes(any *aDocID,
long versionNumber,
char *stockCodes,
char *companyCodes,
char *industryCodes);
void freeWAISDocumentCodes(WAISDocumentCodes *docCodes);
char *writeWAISDocumentCodes(WAISDocumentCodes *docCodes, char *buffer, long *len);
char *readWAISDocumentCodes(WAISDocumentCodes **docCodes, char *buffer);
any *makeWAISTextQuery(DocObj **docs);
DocObj **readWAISTextQuery(any *terms);
void CSTFreeWAISInitResponse(WAISInitResponse *init);
void CSTFreeWAISSearch(WAISSearch *query);
void CSTFreeDocObj(DocObj *doc);
void CSTFreeWAISDocumentHeader(WAISDocumentHeader *header);
void CSTFreeWAISDocumentShortHeader(WAISDocumentShortHeader *header);
void CSTFreeWAISDocumentLongHeader(WAISDocumentLongHeader *header);
void CSTFreeWAISSearchResponse(WAISSearchResponse * response);
void CSTFreeWAISDocumentText(WAISDocumentText *docText);
void CSTFreeWAISDocHeadlines(WAISDocumentHeadlines *docHeadline);
void CSTFreeWAISDocumentCodes(WAISDocumentCodes *docCodes);
void CSTFreeWAISTextQuery(any *query);
/*----------------------------------------------------------------------*/
#endif /* ndef _H_WAIS_protocol_ */
/*
* Routines originally from WMessage.h -- FM
*
*----------------------------------------------------------------------*/
/* WIDE AREA INFORMATION SERVER SOFTWARE
* No guarantees or restrictions. See the readme file for the full standard
* disclaimer.
* 3.26.90
*/
/* wais-message.h
*
* This is the header outside of WAIS Z39.50 messages. The header will be
* printable ascii, so as to be transportable. This header will precede each
* Z39.50 APDU, or zero-length message if it is an ACK or NACK. Be sure to
* change hdr_vers current value if you change the structure of the header.
*
* The characters in the header are case insensitive so that the systems from
* the past that only handle one case can at least read the header.
*
* 7.5.90 HWM - added constants
* 7/5/90 brewster added function prototypes and comments
* 11/30/90 HWM - went to version 2 (inits and typed retrieval)
*/
#ifndef WMESSAGE_H
#define WMESSAGE_H
#ifndef HTUTILS_H
#include <HTUtils.h>
#endif
#include <HTVMS_WaisUI.h>
typedef struct wais_header {
char msg_len[10]; /* length in bytes of following message */
char msg_type; /* type of message: 'z'=Z39.50 APDU,
'a'=ACK, 'n'=NACK */
char hdr_vers; /* version of this header, currently = '2' */
char server[10]; /* name or address of server */
char compression; /* <sp>=no compression, 'u'=unix compress */
char encoding; /* <sp>=no encoding, 'h'=hexize,
'u'=uuencode */
char msg_checksum; /* XOR of every byte of message */
} WAISMessage;
#define HEADER_LENGTH 25 /* number of bytes needed to write a
wais-header (not sizeof(wais_header)) */
#define HEADER_VERSION (long)'2'
/* message type */
#define Z3950 'z'
#define ACK 'a'
#define NAK 'n'
/* compression */
#define NO_COMPRESSION ' '
#define UNIX_COMPRESSION 'u'
/* encoding */
#define NO_ENCODING ' '
#define HEX_ENCODING 'h' /* Swartz 4/3 encoding */
#define IBM_HEXCODING 'i' /* same as h but uses characters acceptable for IBM mainframes */
#define UUENCODE 'u'
void readWAISPacketHeader(char *msgBuffer, WAISMessage * header_struct);
long getWAISPacketLength(WAISMessage * header);
void writeWAISPacketHeader(char *header, long dataLen, long type,
char *server, long compression,
long encoding, long version);
#endif /* ndef WMESSAGE_H */