about summary refs log blame commit diff stats
path: root/WWW/Library/Implementation/HTVMS_WaisProt.h
blob: cea2a325accf6e1ba214a3ae3a3fecba4e2e813f (plain) (tree)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# Copyright (C) 2009, 2010  Roman Zimbelmann <romanz@lavabit.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

if __name__ == '__main__': from __init__ import init; init()
import unittest
from collections import deque

from ranger.ext.iter_tools import *

class TestCases(unittest.TestCase):
	def test_flatten(self):
		def f(x):
			return list(flatten(x))

		self.assertEqual(
			[1,2,3,4,5],
			f([1,2,3,4,5]))
		self.assertEqual(
			[1,2,3,4,5],
			f([1,[2,3],4,5]))
		self.assertEqual(
			[1,2,3,4,5],
			f([[1,[2,3]],4,5]))
		self.assertEqual(
			[],
			f([[[[]]]]))
		self.assertEqual(
			['a', 'b', 'fskldfjl'],
			f(['a', ('b', 'fskldfjl')]))
		self.assertEqual(
			['a', 'b', 'fskldfjl'],
			f(['a', deque(['b', 'fskldfjl'])]))
		self.assertEqual(
			set([3.5, 4.3, 5.2, 6.0]),
			set(f([6.0, set((3.5, 4.3)), (5.2, )])))

	def test_unique(self):
		def u(x):
			return list(unique(x))

		self.assertEqual(
			[1,2,3],
			u([1,2,3]))
		self.assertEqual(
			[1,2,3],
			upre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
/*							HTVMS_WAISProt.h
 *
 *	Adaptation for Lynx by F.Macrides (macrides@sci.wfeb.edu)
 *
 *	31-May-1994 FM	Initial version.
 *
 *----------------------------------------------------------------------*/

/*
 *	Routines originally from WProt.h -- FM
 *
 *----------------------------------------------------------------------*/
/* WIDE AREA INFORMATION SERVER SOFTWARE:
 * No guarantees or restrictions.  See the readme file for the full standard
 * disclaimer.
 *
 *  3.26.90	Harry Morris, morris@think.com
 *  3.30.90  Harry Morris
 *			-	removed chunk code from WAISSearchAPDU,
 *			-	added makeWAISQueryType1Query() and readWAISType1Query() which
 *				replace makeWAISQueryTerms() and makeWAISQueryDocs().
 *  4.11.90  HWM - added definitions of wais element set names
 *  4.14.90  HWM - changed symbol for relevance feedback query from QT_3 to
 *				  QT_RelevanceFeedbackQuery added QT_TextRetrievalQuery as a
 *				  synonym for QT_BooleanQuery
 *				- renamed makeWAISType1Query() to makeWAISTextQuery()
 *				  renamed readWAISType1Query() to readWAISTextQuery()
 *  5.29.90  TS - added CSTFreeWAISFoo functions
 */

#ifndef _H_WAIS_protocol_
#define _H_WAIS_protocol_

#ifndef HTUTILS_H
#include <HTUtils.h>
#endif

#include <HTVMS_WaisUI.h>

/*----------------------------------------------------------------------*/
/* Data types / constants */

/* date factor constants */
#define	DF_INDEPENDENT		1
#define DF_LATER		2
#define DF_EARLIER		3
#define DF_SPECIFIED_RANGE	4

/* chunk types */
#define CT_document		0
#define CT_byte			1
#define CT_line			2
#define CT_paragraph	3

/* relevance feedback query */
#define QT_RelevanceFeedbackQuery	"3"
#define QT_TextRetrievalQuery		QT_BooleanQuery

/* new data tags */
#define DT_UserInformationLength	(data_tag)99
#define	DT_ChunkCode			(data_tag)100
#define	DT_ChunkIDLength		(data_tag)101
#define	DT_ChunkMarker			(data_tag)102
#define	DT_HighlightMarker		(data_tag)103
#define	DT_DeHighlightMarker		(data_tag)104
#define	DT_NewlineCharacters		(data_tag)105
#define	DT_SeedWords			(data_tag)106
#define	DT_DocumentIDChunk		(data_tag)107
#define	DT_ChunkStartID			(data_tag)108
#define	DT_ChunkEndID			(data_tag)109
#define	DT_TextList			(data_tag)110
#define	DT_DateFactor			(data_tag)111
#define	DT_BeginDateRange		(data_tag)112
#define	DT_EndDateRange			(data_tag)113
#define	DT_MaxDocumentsRetrieved	(data_tag)114
#define	DT_SeedWordsUsed		(data_tag)115
#define	DT_DocumentID			(data_tag)116
#define	DT_VersionNumber		(data_tag)117
#define	DT_Score			(data_tag)118
#define	DT_BestMatch			(data_tag)119
#define	DT_DocumentLength		(data_tag)120
#define	DT_Source			(data_tag)121
#define	DT_Date				(data_tag)122
#define	DT_Headline			(data_tag)123
#define	DT_OriginCity			(data_tag)124
#define	DT_PresentStartByte		(data_tag)125
#define	DT_TextLength			(data_tag)126
#define	DT_DocumentText			(data_tag)127
#define	DT_StockCodes			(data_tag)128
#define	DT_CompanyCodes			(data_tag)129
#define	DT_IndustryCodes		(data_tag)130

/* added by harry */
#define DT_DocumentHeaderGroup		(data_tag)150
#define DT_DocumentShortHeaderGroup	(data_tag)151
#define DT_DocumentLongHeaderGroup	(data_tag)152
#define DT_DocumentTextGroup		(data_tag)153
#define DT_DocumentHeadlineGroup	(data_tag)154
#define DT_DocumentCodeGroup		(data_tag)155
#define DT_Lines			(data_tag)131
#define	DT_TYPE_BLOCK			(data_tag)132
#define DT_TYPE				(data_tag)133

/* wais element sets */
#define ES_DocumentHeader		"Document Header"
#define ES_DocumentShortHeader		"Document Short Header"
#define ES_DocumentLongHeader		"Document Long Header"
#define ES_DocumentText			"Document Text"
#define ES_DocumentHeadline		"Document Headline"
#define ES_DocumentCodes		"Document Codes"

typedef struct DocObj {		/* specifies a section of a document */
    any *DocumentID;
    char *Type;
    long ChunkCode;
    union {
	long Pos;
	any *ID;
    } ChunkStart;
    union {
	long Pos;
	any *ID;
    } ChunkEnd;
} DocObj;

/*----------------------------------------------------------------------*/
/* WAIS APDU extensions */

typedef struct WAISInitResponse {
    long ChunkCode;
    long ChunkIDLength;
    char *ChunkMarker;
    char *HighlightMarker;
    char *DeHighlightMarker;
    char *NewlineCharacters;
    /* XXX  need to add UpdateFrequency and Update Time */
} WAISInitResponse;

typedef struct WAISSearch {
    char *SeedWords;
    DocObj **Docs;
    char **TextList;
    long DateFactor;
    char *BeginDateRange;
    char *EndDateRange;
    long MaxDocumentsRetrieved;
} WAISSearch;

typedef struct WAISDocumentHeader {
    any *DocumentID;
    long VersionNumber;
    long Score;
    long BestMatch;
    long DocumentLength;
    long Lines;
    char **Types;
    char *Source;
    char *Date;
    char *Headline;
    char *OriginCity;
} WAISDocumentHeader;

typedef struct WAISDocumentShortHeader {
    any *DocumentID;
    long VersionNumber;
    long Score;
    long BestMatch;
    long DocumentLength;
    long Lines;
} WAISDocumentShortHeader;

typedef struct WAISDocumentLongHeader {
    any *DocumentID;
    long VersionNumber;
    long Score;
    long BestMatch;
    long DocumentLength;
    long Lines;
    char **Types;
    char *Source;
    char *Date;
    char *Headline;
    char *OriginCity;
    char *StockCodes;
    char *CompanyCodes;
    char *IndustryCodes;
} WAISDocumentLongHeader;

typedef struct WAISDocumentText {
    any *DocumentID;
    long VersionNumber;
    any *DocumentText;
} WAISDocumentText;

typedef struct WAISDocumentHeadlines {
    any *DocumentID;
    long VersionNumber;
    char *Source;
    char *Date;
    char *Headline;
    char *OriginCity;
} WAISDocumentHeadlines;

typedef struct WAISDocumentCodes {
    any *DocumentID;
    long VersionNumber;
    char *StockCodes;
    char *CompanyCodes;
    char *IndustryCodes;
} WAISDocumentCodes;

typedef struct WAISSearchResponse {
    char *SeedWordsUsed;
    WAISDocumentHeader **DocHeaders;
    WAISDocumentShortHeader **ShortHeaders;
    WAISDocumentLongHeader **LongHeaders;
    WAISDocumentText **Text;
    WAISDocumentHeadlines **Headlines;
    WAISDocumentCodes **Codes;
    diagnosticRecord **Diagnostics;
} WAISSearchResponse;

/*----------------------------------------------------------------------*/
/* Functions */

char *generate_search_apdu(char *buff,	/* buffer to hold the apdu */
			   long *buff_len,	/* number of bytes written to the buffer */
			   char *seed_words,	/* string of the seed words */
			   char *database_name,
			   DocObj **docobjs,
			   long maxDocsRetrieved);

DocObj *makeDocObjUsingWholeDocument(any *aDocID, char *type);
DocObj *makeDocObjUsingBytes(any *aDocID, char *type, long start, long end);
DocObj *makeDocObjUsingLines(any *aDocID, char *type, long start, long end);
DocObj *makeDocObjUsingParagraphs(any *aDocID, char *type, any *start, any *end);
void freeDocObj(DocObj *doc);

WAISInitResponse *makeWAISInitResponse(long chunkCode, long chunkIDLen,
				       char *chunkMarker, char *highlightMarker,
				       char *deHighlightMarker, char *newLineChars);
void freeWAISInitResponse(WAISInitResponse *init);

WAISSearch *makeWAISSearch(char *seedWords,
			   DocObj **docs,
			   char **textList,
			   long dateFactor,
			   char *beginDateRange,
			   char *endDateRange,
			   long maxDocsRetrieved);
void freeWAISSearch(WAISSearch *query);

WAISDocumentHeader *makeWAISDocumentHeader(any *aDocID,
					   long versionNumber,
					   long score,
					   long bestMatch,
					   long docLen,
					   long lines,
					   char **types,
					   char *source,
					   char *date,
					   char *headline,
					   char *originCity);
void freeWAISDocumentHeader(WAISDocumentHeader *header);
char *writeWAISDocumentHeader(WAISDocumentHeader *header, char *buffer, long *len);
char *readWAISDocumentHeader(WAISDocumentHeader **header, char *buffer);

WAISDocumentShortHeader *makeWAISDocumentShortHeader(any *aDocID,
						     long versionNumber,
						     long score,
						     long bestMatch,
						     long docLen,
						     long lines);
void freeWAISDocumentShortHeader(WAISDocumentShortHeader *header);
char *writeWAISDocumentShortHeader(WAISDocumentShortHeader *header,
				   char *buffer, long *len);
char *readWAISDocumentShortHeader(WAISDocumentShortHeader **header, char *buffer);

WAISDocumentLongHeader *makeWAISDocumentLongHeader(any *aDocID,
						   long versionNumber,
						   long score,
						   long bestMatch,
						   long docLen,
						   long lines,
						   char **types,
						   char *source,
						   char *date, char *headline,
						   char *originCity,
						   char *stockCodes,
						   char *companyCodes,
						   char *industryCodes);
void freeWAISDocumentLongHeader(WAISDocumentLongHeader *header);
char *writeWAISDocumentLongHeader(WAISDocumentLongHeader *header,
				  char *buffer,
				  long *len);
char *readWAISDocumentLongHeader(WAISDocumentLongHeader **header, char *buffer);

WAISSearchResponse *makeWAISSearchResponse(char *seedWordsUsed,
					   WAISDocumentHeader **docHeaders,
					   WAISDocumentShortHeader **shortHeaders,
					   WAISDocumentLongHeader **longHeaders,
					   WAISDocumentText **text, WAISDocumentHeadlines **headlines,
					   WAISDocumentCodes **codes,
					   diagnosticRecord ** diagnostics);
void freeWAISSearchResponse(WAISSearchResponse * response);

WAISDocumentText *makeWAISDocumentText(any *aDocID, long versionNumber,
				       any *documentText);
void freeWAISDocumentText(WAISDocumentText *docText);
char *writeWAISDocumentText(WAISDocumentText *docText, char *buffer, long *len);
char *readWAISDocumentText(WAISDocumentText **docText, char *buffer);

WAISDocumentHeadlines *makeWAISDocumentHeadlines(any *aDocID,
						 long versionNumber,
						 char *source,
						 char *date,
						 char *headline,
						 char *originCity);
void freeWAISDocumentHeadlines(WAISDocumentHeadlines *docHeadline);
char *writeWAISDocumentHeadlines(WAISDocumentHeadlines *docHeadline,
				 char *buffer,
				 long *len);
char *readWAISDocumentHeadlines(WAISDocumentHeadlines **docHeadline, char *buffer);

WAISDocumentCodes *makeWAISDocumentCodes(any *aDocID,
					 long versionNumber,
					 char *stockCodes,
					 char *companyCodes,
					 char *industryCodes);
void freeWAISDocumentCodes(WAISDocumentCodes *docCodes);
char *writeWAISDocumentCodes(WAISDocumentCodes *docCodes, char *buffer, long *len);
char *readWAISDocumentCodes(WAISDocumentCodes **docCodes, char *buffer);

any *makeWAISTextQuery(DocObj **docs);
DocObj **readWAISTextQuery(any *terms);

void CSTFreeWAISInitResponse(WAISInitResponse *init);
void CSTFreeWAISSearch(WAISSearch *query);
void CSTFreeDocObj(DocObj *doc);
void CSTFreeWAISDocumentHeader(WAISDocumentHeader *header);
void CSTFreeWAISDocumentShortHeader(WAISDocumentShortHeader *header);
void CSTFreeWAISDocumentLongHeader(WAISDocumentLongHeader *header);
void CSTFreeWAISSearchResponse(WAISSearchResponse * response);
void CSTFreeWAISDocumentText(WAISDocumentText *docText);
void CSTFreeWAISDocHeadlines(WAISDocumentHeadlines *docHeadline);
void CSTFreeWAISDocumentCodes(WAISDocumentCodes *docCodes);
void CSTFreeWAISTextQuery(any *query);

/*----------------------------------------------------------------------*/

#endif /* ndef _H_WAIS_protocol_ */

/*
 *	Routines originally from WMessage.h -- FM
 *
 *----------------------------------------------------------------------*/
/* WIDE AREA INFORMATION SERVER SOFTWARE
 * No guarantees or restrictions.  See the readme file for the full standard
 * disclaimer.
 * 3.26.90
 */

/* wais-message.h
 *
 * This is the header outside of WAIS Z39.50 messages.  The header will be
 * printable ascii, so as to be transportable.  This header will precede each
 * Z39.50 APDU, or zero-length message if it is an ACK or NACK.  Be sure to
 * change hdr_vers current value if you change the structure of the header.
 *
 * The characters in the header are case insensitive so that the systems from
 * the past that only handle one case can at least read the header.
 *
 * 7.5.90 HWM - added constants
 * 7/5/90 brewster added function prototypes and comments
 * 11/30/90 HWM - went to version 2 (inits and typed retrieval)
 */

#ifndef WMESSAGE_H
#define WMESSAGE_H

#ifndef HTUTILS_H
#include <HTUtils.h>
#endif

#include <HTVMS_WaisUI.h>

typedef struct wais_header {
    char msg_len[10];		/* length in bytes of following message */
    char msg_type;		/* type of message: 'z'=Z39.50 APDU,
				   'a'=ACK, 'n'=NACK */
    char hdr_vers;		/* version of this header, currently = '2' */
    char server[10];		/* name or address of server */
    char compression;		/* <sp>=no compression, 'u'=unix compress */
    char encoding;		/* <sp>=no encoding, 'h'=hexize,
				   'u'=uuencode */
    char msg_checksum;		/* XOR of every byte of message */
} WAISMessage;

#define HEADER_LENGTH	25	/* number of bytes needed to write a
				   wais-header (not sizeof(wais_header)) */

#define HEADER_VERSION	(long)'2'

/* message type */
#define Z3950			'z'
#define ACK			'a'
#define	NAK			'n'

/* compression */
#define NO_COMPRESSION		' '
#define UNIX_COMPRESSION	'u'

/* encoding */
#define NO_ENCODING		' '
#define HEX_ENCODING		'h'	/* Swartz 4/3 encoding */
#define IBM_HEXCODING		'i'	/* same as h but uses characters acceptable for IBM mainframes */
#define UUENCODE		'u'

void readWAISPacketHeader(char *msgBuffer, WAISMessage * header_struct);
long getWAISPacketLength(WAISMessage * header);
void writeWAISPacketHeader(char *header, long dataLen, long type,
			   char *server, long compression,
			   long encoding, long version);

#endif /* ndef WMESSAGE_H */