aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.md49
-rw-r--r--app.py44
-rw-r--r--requirements.txt1
-rw-r--r--sphinx/__init__.py0
-rw-r--r--sphinx/sphinx.py44
-rw-r--r--sphinx/sphinxapi.py1315
-rw-r--r--sphinx_search.conf5
7 files changed, 38 insertions, 1420 deletions
diff --git a/README.md b/README.md
index f8bec01..3d63946 100644
--- a/README.md
+++ b/README.md
@@ -18,14 +18,13 @@ file)!
## Sphinx-search / Manticore-search
-You can use [Sphinx-search](http://sphinxsearch.com/) but it's recommand to use
-[Manticore-search](https://manticoresearch.com/) since the last version of
-Sphinx-search is ditribued in closed-source instead of open-source (for
-version 3.x).
+You must use [Manticore-search](https://manticoresearch.com/) because of the
+usage of the JSON search API in the searx engines.
-All explication is for Manticore-search for the moment but at many time the
-term `sphinx` is used in code because Manticore-search want to keep a
-compatibility with Sphinx-search.
+But you can use [Sphinx-search](http://sphinxsearch.com/) if you don't want to
+use the JSON search API. You need to know, as of January 2019, the last
+version of Sphinx-search is distribued in closed-source instead of open-source
+(for versions 3.x)
# Configuration
@@ -34,12 +33,10 @@ compatibility with Sphinx-search.
The database used for this project is PostgreSQL, you can update login
information in `config.py` file.
-## Sphinx-search / Manticore-search
+## Manticore-search
The configuration for this is in `sphinx_search.conf` file. For update this
-file please view documentation of
-[Sphinx-search](http://sphinxsearch.com/docs/manual-2.3.2.html) or
-[Manticore-search](https://docs.manticoresearch.com).
+file please view documentation [Manticore-search](https://docs.manticoresearch.com).
Keep in mind you must keep up to date the file `config.py` in accordance with
the `sphinx_search.conf` file.
@@ -91,12 +88,36 @@ You can now launch the server!
python app.py
```
-For start searching send `GET` request to the following adresse (without `<` and
-`>`):
+For start searching send a `POST` request with the manticoresearch json API,
+for example:
+
+```
+http POST 'http://localhost:8080/json/search' < mysearch.json
+```
+
+This is the content of the `mysearch.json`:
+
```
-127.0.0.1:5000/?search=<search terms>&index=<indexed website>
+{
+ "index": "neodarznet",
+ "query": { "match": { "content": "Livet" } },
+ "highlight":
+ {
+ "fields":
+ {
+ "content": {},
+ "url": {},
+ "title": {}
+ },
+ "pre_tags": "_",
+ "post_tags": "_",
+ }
+}
```
+You can find more information about the HTTP sear API avaiblable in the
+[Manticores-earch documentation](https://docs.manticoresearch.com/latest/html/httpapi_reference.html)
+
Resultat are in json format. If you whant to know witch website is indexed,
search in the file [sphinx_search.conf](https://git.khaganat.net/neodarz/khanindexer/blob/master/sphinx_search.conf)
all the line who start by `index`.
diff --git a/app.py b/app.py
index 66ca6c4..431c2b0 100644
--- a/app.py
+++ b/app.py
@@ -3,34 +3,10 @@ import sys
from scrapy.crawler import CrawlerProcess
from scrapy import spiderloader
from scrapy.utils.project import get_project_settings
-from flask import Flask, request, jsonify
-import json
-
-from sphinx import sphinx
from database.models import Page, db
-import config
-
-app = Flask(__name__)
-
-@app.route("/", methods=['GET', 'POST'])
-def search():
- query = request.args.get('search')
- index = request.args.get('index')
- if query != '' and query != None and index != '' and index != None:
- response = sphinx.search(query, index)
- elif query is None:
- response = {'error': 1, 'msg': 'Use `search` attribute for make a search'}
- elif index == None:
- response = {'error': 1, 'msg': 'Use `index` attribute to precise an index'}
- elif index == '':
- response = {'error': 1, 'msg': '`index` cannot be null'}
- else:
- response = {'error': 1, 'msg': '`search` cannot be null'}
- return jsonify(response)
-
-def crawl():
+def main():
try:
db.create_tables(Page.__subclasses__())
settings = get_project_settings()
@@ -42,21 +18,5 @@ def crawl():
except Exception as e:
print(e)
-def main():
-
- app.run(debug=True, use_reloader=True)
-
-def show_help():
- print("Launch server => "+str(sys.argv[0]))
- print("Launch all crawler => "+str(sys.argv[0])+" crawl")
-
if __name__ == '__main__':
- if len(sys.argv) == 1:
- main()
- elif len(sys.argv) == 2:
- if sys.argv[1] == "crawl":
- crawl()
- else:
- show_help()
- else:
- show_help()
+ main()
diff --git a/requirements.txt b/requirements.txt
index 6607cd0..2c2a474 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
Scrapy==1.5.1
-Flask==1.0.2
peewee==3.8.1
psycopg2==2.7.6.1
diff --git a/sphinx/__init__.py b/sphinx/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/sphinx/__init__.py
+++ /dev/null
diff --git a/sphinx/sphinx.py b/sphinx/sphinx.py
deleted file mode 100644
index 11d3fee..0000000
--- a/sphinx/sphinx.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from .sphinxapi import *
-
-import config
-
-sphinxclient = SphinxClient()
-
-sphinxclient.SetServer(host=config.SPHINX_HOST, port=config.SPHINX_PORT)
-
-def search(request, index):
- """
- Simple sphinx seach function, return a array of all documents matching the
- search terms.
- """
- status = sphinxclient.Status()
- response = {'error': 1, 'msg': 'Sphinx server not available'}
- if status != None:
- res = sphinxclient.Query(request, index)
-
- opts = {'before_match': '', 'after_match':'', 'chunk_separator': '...', 'limit': 400, 'around': 15}
-
- response = []
-
- if res != None:
- if 'matches' in res:
- n = 1
- for match in res['matches']:
- attrsdump = ''
- for attr in res['attrs']:
- attrname = attr[0]
- attrtype = attr[1]
- if attrname != "content":
- value = match['attrs'][attrname]
- if attrtype==SPH_ATTR_TIMESTAMP:
- value = time.strftime ( '%Y-%m-%d %H:%M:%S', time.localtime(value) )
- attrsdump = '%s, \'%s\'=\'%s\'' % ( attrsdump, attrname, value)
- docs = []
- docs.append(''.join([line.strip('\n') for line in match['attrs']['content']]))
- res_excerpts = sphinxclient.BuildExcerpts(index=index, docs=docs, opts=opts, words=request)
- response.append({'id': match['id'], 'weight': match['weight'], 'url': match['attrs']['url'], 'title': match['attrs']['title'], 'excerpts': res_excerpts})
- n += 1
- else:
- response = {'error': 1, 'msg': 'Maybe `'+str(index)+'` is not an index in the database?'}
-
- return response
diff --git a/sphinx/sphinxapi.py b/sphinx/sphinxapi.py
deleted file mode 100644
index 62a88cb..0000000
--- a/sphinx/sphinxapi.py
+++ /dev/null
@@ -1,1315 +0,0 @@
-#
-# $Id$
-#
-# Python version of Sphinx searchd client (Python API)
-#
-# Copyright (c) 2006, Mike Osadnik
-# Copyright (c) 2006-2016, Andrew Aksyonoff
-# Copyright (c) 2008-2016, Sphinx Technologies Inc
-# All rights reserved
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU Library General Public License. You should
-# have received a copy of the LGPL license along with this program; if you
-# did not, you can find it at http://www.gnu.org/
-#
-# WARNING!!!
-#
-# As of 2015, we strongly recommend to use either SphinxQL or REST APIs
-# rather than the native SphinxAPI.
-#
-# While both the native SphinxAPI protocol and the existing APIs will
-# continue to exist, and perhaps should not even break (too much), exposing
-# all the new features via multiple different native API implementations
-# is too much of a support complication for us.
-#
-# That said, you're welcome to overtake the maintenance of any given
-# official API, and remove this warning ;)
-#
-
-from __future__ import print_function
-import sys
-import select
-import socket
-import re
-from struct import *
-
-if sys.version_info > (3,):
- long = int
- text_type = str
-else:
- text_type = unicode
-
-# known searchd commands
-SEARCHD_COMMAND_SEARCH = 0
-SEARCHD_COMMAND_EXCERPT = 1
-SEARCHD_COMMAND_UPDATE = 2
-SEARCHD_COMMAND_KEYWORDS = 3
-SEARCHD_COMMAND_PERSIST = 4
-SEARCHD_COMMAND_STATUS = 5
-SEARCHD_COMMAND_FLUSHATTRS = 7
-
-# current client-side command implementation versions
-VER_COMMAND_SEARCH = 0x120
-VER_COMMAND_EXCERPT = 0x104
-VER_COMMAND_UPDATE = 0x103
-VER_COMMAND_KEYWORDS = 0x100
-VER_COMMAND_STATUS = 0x101
-VER_COMMAND_FLUSHATTRS = 0x100
-
-# known searchd status codes
-SEARCHD_OK = 0
-SEARCHD_ERROR = 1
-SEARCHD_RETRY = 2
-SEARCHD_WARNING = 3
-
-# known match modes
-SPH_MATCH_ALL = 0
-SPH_MATCH_ANY = 1
-SPH_MATCH_PHRASE = 2
-SPH_MATCH_BOOLEAN = 3
-SPH_MATCH_EXTENDED = 4
-SPH_MATCH_FULLSCAN = 5
-SPH_MATCH_EXTENDED2 = 6
-
-# known ranking modes (extended2 mode only)
-SPH_RANK_PROXIMITY_BM25 = 0 # default mode, phrase proximity major factor and BM25 minor one
-SPH_RANK_BM25 = 1 # statistical mode, BM25 ranking only (faster but worse quality)
-SPH_RANK_NONE = 2 # no ranking, all matches get a weight of 1
-SPH_RANK_WORDCOUNT = 3 # simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts
-SPH_RANK_PROXIMITY = 4
-SPH_RANK_MATCHANY = 5
-SPH_RANK_FIELDMASK = 6
-SPH_RANK_SPH04 = 7
-SPH_RANK_EXPR = 8
-SPH_RANK_TOTAL = 9
-
-# known sort modes
-SPH_SORT_RELEVANCE = 0
-SPH_SORT_ATTR_DESC = 1
-SPH_SORT_ATTR_ASC = 2
-SPH_SORT_TIME_SEGMENTS = 3
-SPH_SORT_EXTENDED = 4
-SPH_SORT_EXPR = 5
-
-# known filter types
-SPH_FILTER_VALUES = 0
-SPH_FILTER_RANGE = 1
-SPH_FILTER_FLOATRANGE = 2
-SPH_FILTER_STRING = 3
-SPH_FILTER_STRING_LIST = 6
-
-# known attribute types
-SPH_ATTR_NONE = 0
-SPH_ATTR_INTEGER = 1
-SPH_ATTR_TIMESTAMP = 2
-SPH_ATTR_ORDINAL = 3
-SPH_ATTR_BOOL = 4
-SPH_ATTR_FLOAT = 5
-SPH_ATTR_BIGINT = 6
-SPH_ATTR_STRING = 7
-SPH_ATTR_FACTORS = 1001
-SPH_ATTR_MULTI = long(0X40000001)
-SPH_ATTR_MULTI64 = long(0X40000002)
-
-SPH_ATTR_TYPES = (SPH_ATTR_NONE,
- SPH_ATTR_INTEGER,
- SPH_ATTR_TIMESTAMP,
- SPH_ATTR_ORDINAL,
- SPH_ATTR_BOOL,
- SPH_ATTR_FLOAT,
- SPH_ATTR_BIGINT,
- SPH_ATTR_STRING,
- SPH_ATTR_MULTI,
- SPH_ATTR_MULTI64)
-
-# known grouping functions
-SPH_GROUPBY_DAY = 0
-SPH_GROUPBY_WEEK = 1
-SPH_GROUPBY_MONTH = 2
-SPH_GROUPBY_YEAR = 3
-SPH_GROUPBY_ATTR = 4
-SPH_GROUPBY_ATTRPAIR = 5
-
-
-class SphinxClient:
- def __init__ (self):
- """
- Create a new client object, and fill defaults.
- """
- self._host = 'localhost' # searchd host (default is "localhost")
- self._port = 9312 # searchd port (default is 9312)
- self._path = None # searchd unix-domain socket path
- self._socket = None
- self._offset = 0 # how much records to seek from result-set start (default is 0)
- self._limit = 20 # how much records to return from result-set starting at offset (default is 20)
- self._mode = SPH_MATCH_EXTENDED2 # query matching mode (default is SPH_MATCH_EXTENDED2)
- self._weights = [] # per-field weights (default is 1 for all fields)
- self._sort = SPH_SORT_RELEVANCE # match sorting mode (default is SPH_SORT_RELEVANCE)
- self._sortby = bytearray() # attribute to sort by (defualt is "")
- self._min_id = 0 # min ID to match (default is 0)
- self._max_id = 0 # max ID to match (default is UINT_MAX)
- self._filters = [] # search filters
- self._groupby = bytearray() # group-by attribute name
- self._groupfunc = SPH_GROUPBY_DAY # group-by function (to pre-process group-by attribute value with)
- self._groupsort = str_bytes('@group desc') # group-by sorting clause (to sort groups in result set with)
- self._groupdistinct = bytearray() # group-by count-distinct attribute
- self._maxmatches = 1000 # max matches to retrieve
- self._cutoff = 0 # cutoff to stop searching at
- self._retrycount = 0 # distributed retry count
- self._retrydelay = 0 # distributed retry delay
- self._anchor = {} # geographical anchor point
- self._indexweights = {} # per-index weights
- self._ranker = SPH_RANK_PROXIMITY_BM25 # ranking mode
- self._rankexpr = bytearray() # ranking expression for SPH_RANK_EXPR
- self._maxquerytime = 0 # max query time, milliseconds (default is 0, do not limit)
- self._timeout = 1.0 # connection timeout
- self._fieldweights = {} # per-field-name weights
- self._overrides = {} # per-query attribute values overrides
- self._select = str_bytes('*') # select-list (attributes or expressions, with optional aliases)
- self._query_flags = SetBit ( 0, 6, True ) # default idf=tfidf_normalized
- self._predictedtime = 0 # per-query max_predicted_time
- self._outerorderby = bytearray() # outer match sort by
- self._outeroffset = 0 # outer offset
- self._outerlimit = 0 # outer limit
- self._hasouter = False # sub-select enabled
- self._tokenfilterlibrary = bytearray() # token_filter plugin library name
- self._tokenfiltername = bytearray() # token_filter plugin name
- self._tokenfilteropts = bytearray() # token_filter plugin options
-
- self._error = '' # last error message
- self._warning = '' # last warning message
- self._reqs = [] # requests array for multi-query
-
- def __del__ (self):
- if self._socket:
- self._socket.close()
-
-
- def GetLastError (self):
- """
- Get last error message (string).
- """
- return self._error
-
-
- def GetLastWarning (self):
- """
- Get last warning message (string).
- """
- return self._warning
-
-
- def SetServer (self, host, port = None):
- """
- Set searchd server host and port.
- """
- assert(isinstance(host, str))
- if host.startswith('/'):
- self._path = host
- return
- elif host.startswith('unix://'):
- self._path = host[7:]
- return
- self._host = host
- if isinstance(port, int):
- assert(port>0 and port<65536)
- self._port = port
- self._path = None
-
- def SetConnectTimeout ( self, timeout ):
- """
- Set connection timeout ( float second )
- """
- assert (isinstance(timeout, float))
- # set timeout to 0 make connaection non-blocking that is wrong so timeout got clipped to reasonable minimum
- self._timeout = max ( 0.001, timeout )
-
- def _Connect (self):
- """
- INTERNAL METHOD, DO NOT CALL. Connects to searchd server.
- """
- if self._socket:
- # we have a socket, but is it still alive?
- sr, sw, _ = select.select ( [self._socket], [self._socket], [], 0 )
-
- # this is how alive socket should look
- if len(sr)==0 and len(sw)==1:
- return self._socket
-
- # oops, looks like it was closed, lets reopen
- self._socket.close()
- self._socket = None
-
- try:
- if self._path:
- af = socket.AF_UNIX
- addr = self._path
- desc = self._path
- else:
- af = socket.AF_INET
- addr = ( self._host, self._port )
- desc = '%s;%s' % addr
- sock = socket.socket ( af, socket.SOCK_STREAM )
- sock.settimeout ( self._timeout )
- sock.connect ( addr )
- except socket.error as msg:
- if sock:
- sock.close()
- self._error = 'connection to %s failed (%s)' % ( desc, msg )
- return
-
- v = unpack('>L', sock.recv(4))[0]
- if v<1:
- sock.close()
- self._error = 'expected searchd protocol version, got %s' % v
- return
-
- # all ok, send my version
- sock.send(pack('>L', 1))
- return sock
-
-
- def _GetResponse (self, sock, client_ver):
- """
- INTERNAL METHOD, DO NOT CALL. Gets and checks response packet from searchd server.
- """
- (status, ver, length) = unpack('>2HL', sock.recv(8))
- response = bytearray()
- left = length
- while left>0:
- chunk = sock.recv(left)
- if chunk:
- response += chunk
- left -= len(chunk)
- else:
- break
-
- if not self._socket:
- sock.close()
-
- # check response
- read = len(response)
- if not response or read!=length:
- if length:
- self._error = 'failed to read searchd response (status=%s, ver=%s, len=%s, read=%s)' \
- % (status, ver, length, read)
- else:
- self._error = 'received zero-sized searchd response'
- return None
-
- # check status
- if status==SEARCHD_WARNING:
- wend = 4 + unpack ( '>L', response[0:4] )[0]
- self._warning = bytes_str(response[4:wend])
- return response[wend:]
-
- if status==SEARCHD_ERROR:
- self._error = 'searchd error: ' + bytes_str(response[4:])
- return None
-
- if status==SEARCHD_RETRY:
- self._error = 'temporary searchd error: ' + bytes_str(response[4:])
- return None
-
- if status!=SEARCHD_OK:
- self._error = 'unknown status code %d' % status
- return None
-
- # check version
- if ver<client_ver:
- self._warning = 'searchd command v.%d.%d older than client\'s v.%d.%d, some options might not work' \
- % (ver>>8, ver&0xff, client_ver>>8, client_ver&0xff)
-
- return response
-
-
- def _Send ( self, sock, req ):
- """
- INTERNAL METHOD, DO NOT CALL. send request to searchd server.
- """
- total = 0
- while True:
- sent = sock.send ( req[total:] )
- if sent<=0:
- break
-
- total = total + sent
-
- return total
-
-
- def SetLimits (self, offset, limit, maxmatches=0, cutoff=0):
- """
- Set offset and count into result set, and optionally set max-matches and cutoff limits.
- """
- assert ( type(offset) in [int,long] and 0<=offset<16777216 )
- assert ( type(limit) in [int,long] and 0<limit<16777216 )
- assert(maxmatches>=0)
- self._offset = offset
- self._limit = limit
- if maxmatches>0:
- self._maxmatches = maxmatches
- if cutoff>=0:
- self._cutoff = cutoff
-
-
- def SetMaxQueryTime (self, maxquerytime):
- """
- Set maximum query time, in milliseconds, per-index. 0 means 'do not limit'.
- """
- assert(isinstance(maxquerytime,int) and maxquerytime>0)
- self._maxquerytime = maxquerytime
-
-
- def SetMatchMode (self, mode):
- """
- Set matching mode.
- """
- print('DEPRECATED: Do not call this method or, even better, use SphinxQL instead of an API', file=sys.stderr)
- assert(mode in [SPH_MATCH_ALL, SPH_MATCH_ANY, SPH_MATCH_PHRASE, SPH_MATCH_BOOLEAN, SPH_MATCH_EXTENDED, SPH_MATCH_FULLSCAN, SPH_MATCH_EXTENDED2])
- self._mode = mode
-
-
- def SetRankingMode ( self, ranker, rankexpr='' ):
- """
- Set ranking mode.
- """
- assert(ranker>=0 and ranker<SPH_RANK_TOTAL)
- self._ranker = ranker
- self._rankexpr = str_bytes(rankexpr)
-
-
- def SetSortMode ( self, mode, clause='' ):
- """
- Set sorting mode.
- """
- assert ( mode in [SPH_SORT_RELEVANCE, SPH_SORT_ATTR_DESC, SPH_SORT_ATTR_ASC, SPH_SORT_TIME_SEGMENTS, SPH_SORT_EXTENDED, SPH_SORT_EXPR] )
- assert ( isinstance ( clause, (str,text_type) ) )
- self._sort = mode
- self._sortby = str_bytes(clause)
-
-
- def SetFieldWeights (self, weights):
- """
- Bind per-field weights by name; expects (name,field_weight) dictionary as argument.
- """
- assert(isinstance(weights,dict))
- for key,val in list(weights.items()):
- assert(isinstance(key,str))
- AssertUInt32 ( val )
- self._fieldweights = weights
-
-
- def SetIndexWeights (self, weights):
- """
- Bind per-index weights by name; expects (name,index_weight) dictionary as argument.
- """
- assert(isinstance(weights,dict))
- for key,val in list(weights.items()):
- assert(isinstance(key,str))
- AssertUInt32(val)
- self._indexweights = weights
-
-
- def SetIDRange (self, minid, maxid):
- """
- Set IDs range to match.
- Only match records if document ID is beetwen $min and $max (inclusive).
- """
- assert(isinstance(minid, (int, long)))
- assert(isinstance(maxid, (int, long)))
- assert(minid<=maxid)
- self._min_id = minid
- self._max_id = maxid
-
-
- def SetFilter ( self, attribute, values, exclude=0 ):
- """
- Set values set filter.
- Only match records where 'attribute' value is in given 'values' set.
- """
- assert(isinstance(attribute, str))
- assert iter(values)
-
- for value in values:
- AssertInt32 ( value )
-
- self._filters.append ( { 'type':SPH_FILTER_VALUES, 'attr':attribute, 'exclude':exclude, 'values':values } )
-
-
- def SetFilterString ( self, attribute, value, exclude=0 ):
- """
- Set string filter.
- Only match records where 'attribute' value is equal
- """
- assert(isinstance(attribute, str))
- assert(isinstance(value, str))
-
-
- self._filters.append ( { 'type':SPH_FILTER_STRING, 'attr':attribute, 'exclude':exclude, 'value':value } )
-
-
- def SetFilterStringList ( self, attribute, value, exclude=0 ):
- """
- Set string list filter.
- """
- assert(isinstance(attribute, str))
- assert(iter(value))
-
- for v in value:
- assert(isinstance(v, str))
-
- self._filters.append ( { 'type':SPH_FILTER_STRING_LIST, 'attr':attribute, 'exclude':exclude, 'values':value } )
-
-
- def SetFilterRange (self, attribute, min_, max_, exclude=0 ):
- """
- Set range filter.
- Only match records if 'attribute' value is beetwen 'min_' and 'max_' (inclusive).
- """
- assert(isinstance(attribute, str))
- AssertInt32(min_)
- AssertInt32(max_)
- assert(min_<=max_)
-
- self._filters.append ( { 'type':SPH_FILTER_RANGE, 'attr':attribute, 'exclude':exclude, 'min':min_, 'max':max_ } )
-
-
- def SetFilterFloatRange (self, attribute, min_, max_, exclude=0 ):
- assert(isinstance(attribute,str))
- assert(isinstance(min_,float))
- assert(isinstance(max_,float))
- assert(min_ <= max_)
- self._filters.append ( {'type':SPH_FILTER_FLOATRANGE, 'attr':attribute, 'exclude':exclude, 'min':min_, 'max':max_} )
-
-
- def SetGeoAnchor (self, attrlat, attrlong, latitude, longitude):
- assert(isinstance(attrlat,str))
- assert(isinstance(attrlong,str))
- assert(isinstance(latitude,float))
- assert(isinstance(longitude,float))
- self._anchor['attrlat'] = attrlat
- self._anchor['attrlong'] = attrlong
- self._anchor['lat'] = latitude
- self._anchor['long'] = longitude
-
-
- def SetGroupBy ( self, attribute, func, groupsort='@group desc' ):
- """
- Set grouping attribute and function.
- """
- assert(isinstance(attribute, (str,text_type)))
- assert(func in [SPH_GROUPBY_DAY, SPH_GROUPBY_WEEK, SPH_GROUPBY_MONTH, SPH_GROUPBY_YEAR, SPH_GROUPBY_ATTR, SPH_GROUPBY_ATTRPAIR] )
- assert(isinstance(groupsort, (str,text_type)))
-
- self._groupby = str_bytes(attribute)
- self._groupfunc = func
- self._groupsort = str_bytes(groupsort)
-
-
- def SetGroupDistinct (self, attribute):
- assert(isinstance(attribute,(str,text_type)))
- self._groupdistinct = str_bytes(attribute)
-
-
- def SetRetries (self, count, delay=0):
- assert(isinstance(count,int) and count>=0)
- assert(isinstance(delay,int) and delay>=0)
- self._retrycount = count
- self._retrydelay = delay
-
-
- def SetOverride (self, name, type, values):
- print('DEPRECATED: Do not call this method. Use SphinxQL REMAP() function instead.', file=sys.stderr)
- assert(isinstance(name, str))
- assert(type in SPH_ATTR_TYPES)
- assert(isinstance(values, dict))
-
- self._overrides[name] = {'name': name, 'type': type, 'values': values}
-
- def SetSelect (self, select):
- assert(isinstance(select, (str,text_type)))
- self._select = str_bytes(select)
-
- def SetQueryFlag ( self, name, value ):
- known_names = [ "reverse_scan", "sort_method", "max_predicted_time", "boolean_simplify", "idf", "global_idf" ]
- flags = { "reverse_scan":[0, 1], "sort_method":["pq", "kbuffer"],"max_predicted_time":[0], "boolean_simplify":[True, False], "idf":["normalized", "plain", "tfidf_normalized", "tfidf_unnormalized"], "global_idf":[True, False] }
- assert ( name in known_names )
- assert ( value in flags[name] or ( name=="max_predicted_time" and isinstance(value, (int, long)) and value>=0))
-
- if name=="reverse_scan":
- self._query_flags = SetBit ( self._query_flags, 0, value==1 )
- if name=="sort_method":
- self._query_flags = SetBit ( self._query_flags, 1, value=="kbuffer" )
- if name=="max_predicted_time":
- self._query_flags = SetBit ( self._query_flags, 2, value>0 )
- self._predictedtime = int(value)
- if name=="boolean_simplify":
- self._query_flags= SetBit ( self._query_flags, 3, value )
- if name=="idf" and ( value=="plain" or value=="normalized" ) :
- self._query_flags = SetBit ( self._query_flags, 4, value=="plain" )
- if name=="global_idf":
- self._query_flags= SetBit ( self._query_flags, 5, value )
- if name=="idf" and ( value=="tfidf_normalized" or value=="tfidf_unnormalized" ) :
- self._query_flags = SetBit ( self._query_flags, 6, value=="tfidf_normalized" )
-
- def SetOuterSelect ( self, orderby, offset, limit ):
- assert(isinstance(orderby, (str,text_type)))
- assert(isinstance(offset, (int, long)))
- assert(isinstance(limit, (int, long)))
- assert ( offset>=0 )
- assert ( limit>0 )
-
- self._outerorderby = str_bytes(orderby)
- self._outeroffset = offset
- self._outerlimit = limit
- self._hasouter = True
-
- def SetTokenFilter ( self, library, name, opts='' ):
- assert(isinstance(library, str))
- assert(isinstance(name, str))
- assert(isinstance(opts, str))
-
- self._tokenfilterlibrary = str_bytes(library)
- self._tokenfiltername = str_bytes(name)
- self._tokenfilteropts = str_bytes(opts)
-
- def ResetOverrides (self):
- self._overrides = {}
-
-
- def ResetFilters (self):
- """
- Clear all filters (for multi-queries).
- """
- self._filters = []
- self._anchor = {}
-
-
- def ResetGroupBy (self):
- """
- Clear groupby settings (for multi-queries).
- """
- self._groupby = bytearray()
- self._groupfunc = SPH_GROUPBY_DAY
- self._groupsort = str_bytes('@group desc')
- self._groupdistinct = bytearray()
-
- def ResetQueryFlag (self):
- self._query_flags = SetBit ( 0, 6, True ) # default idf=tfidf_normalized
- self._predictedtime = 0
-
- def ResetOuterSelect (self):
- self._outerorderby = bytearray()
- self._outeroffset = 0
- self._outerlimit = 0
- self._hasouter = False
-
- def Query (self, query, index='*', comment=''):
- """
- Connect to searchd server and run given search query.
- Returns None on failure; result set hash on success (see documentation for details).
- """
- assert(len(self._reqs)==0)
- self.AddQuery(query,index,comment)
- results = self.RunQueries()
- self._reqs = [] # we won't re-run erroneous batch
-
- if not results or len(results)==0:
- return None
- self._error = results[0]['error']
- self._warning = results[0]['warning']
- if results[0]['status'] == SEARCHD_ERROR:
- return None
- return results[0]
-
-
- def AddQuery (self, query, index='*', comment=''):
- """
- Add query to batch.
- """
- # build request
- req = bytearray()
- req.extend(pack('>5L', self._query_flags, self._offset, self._limit, self._mode, self._ranker))
- if self._ranker==SPH_RANK_EXPR:
- req.extend(pack('>L', len(self._rankexpr)))
- req.extend(self._rankexpr)
- req.extend(pack('>L', self._sort))
- req.extend(pack('>L', len(self._sortby)))
- req.extend(self._sortby)
-
- query = str_bytes(query)
- assert(isinstance(query,bytearray))
-
- req.extend(pack('>L', len(query)))
- req.extend(query)
-
- req.extend(pack('>L', len(self._weights)))
- for w in self._weights:
- req.extend(pack('>L', w))
- index = str_bytes(index)
- assert(isinstance(index,bytearray))
- req.extend(pack('>L', len(index)))
- req.extend(index)
- req.extend(pack('>L',1)) # id64 range marker
- req.extend(pack('>Q', self._min_id))
- req.extend(pack('>Q', self._max_id))
-
- # filters
- req.extend ( pack ( '>L', len(self._filters) ) )
- for f in self._filters:
- attr = str_bytes(f['attr'])
- req.extend ( pack ( '>L', len(f['attr'])) + attr)
- filtertype = f['type']
- req.extend ( pack ( '>L', filtertype))
- if filtertype == SPH_FILTER_VALUES:
- req.extend ( pack ('>L', len(f['values'])))
- for val in f['values']:
- req.extend ( pack ('>q', val))
- elif filtertype == SPH_FILTER_RANGE:
- req.extend ( pack ('>2q', f['min'], f['max']))
- elif filtertype == SPH_FILTER_FLOATRANGE:
- req.extend ( pack ('>2f', f['min'], f['max']))
- elif filtertype == SPH_FILTER_STRING:
- val = str_bytes(f['value'])
- req.extend ( pack ( '>L', len(val) ) )
- req.extend ( val )
- elif filtertype == SPH_FILTER_STRING_LIST:
- req.extend ( pack ('>L', len(f['values'])))
- for sval in f['values']:
- val = str_bytes( sval )
- req.extend ( pack ( '>L', len(val) ) )
- req.extend(val)
- req.extend ( pack ( '>L', f['exclude'] ) )
-
- # group-by, max-matches, group-sort
- req.extend ( pack ( '>2L', self._groupfunc, len(self._groupby) ) )
- req.extend ( self._groupby )
- req.extend ( pack ( '>2L', self._maxmatches, len(self._groupsort) ) )
- req.extend ( self._groupsort )
- req.extend ( pack ( '>LLL', self._cutoff, self._retrycount, self._retrydelay))
- req.extend ( pack ( '>L', len(self._groupdistinct)))
- req.extend ( self._groupdistinct)
-
- # anchor point
- if len(self._anchor) == 0:
- req.extend ( pack ('>L', 0))
- else:
- attrlat, attrlong = str_bytes(self._anchor['attrlat']), str_bytes(self._anchor['attrlong'])
- latitude, longitude = self._anchor['lat'], self._anchor['long']
- req.extend ( pack ('>L', 1))
- req.extend ( pack ('>L', len(attrlat)) + attrlat)
- req.extend ( pack ('>L', len(attrlong)) + attrlong)
- req.extend ( pack ('>f', latitude) + pack ('>f', longitude))
-
- # per-index weights
- req.extend ( pack ('>L',len(self._indexweights)))
- for indx,weight in list(self._indexweights.items()):
- indx = str_bytes(indx)
- req.extend ( pack ('>L',len(indx)) + indx + pack ('>L',weight))
-
- # max query time
- req.extend ( pack ('>L', self._maxquerytime) )
-
- # per-field weights
- req.extend ( pack ('>L',len(self._fieldweights) ) )
- for field,weight in list(self._fieldweights.items()):
- field = str_bytes(field)
- req.extend ( pack ('>L',len(field)) + field + pack ('>L',weight) )
-
- # comment
- comment = str_bytes(comment)
- req.extend ( pack('>L',len(comment)) + comment )
-
- # attribute overrides
- req.extend ( pack('>L', len(self._overrides)) )
- for v in list(self._overrides.values()):
- name = str_bytes(v['name'])
- req.extend ( ( pack('>L', len(name)), name ) )
- req.extend ( pack('>LL', v['type'], len(v['values'])) )
- for id, value in v['values'].items():
- req.extend ( pack('>Q', id) )
- if v['type'] == SPH_ATTR_FLOAT:
- req.extend ( pack('>f', value) )
- elif v['type'] == SPH_ATTR_BIGINT:
- req.extend ( pack('>q', value) )
- else:
- req.extend ( pack('>l', value) )
-
- # select-list
- req.extend ( pack('>L', len(self._select)) )
- req.extend ( self._select )
- if self._predictedtime>0:
- req.extend ( pack('>L', self._predictedtime ) )
-
- # outer
- req.extend ( pack('>L',len(self._outerorderby)) + self._outerorderby )
- req.extend ( pack ( '>2L', self._outeroffset, self._outerlimit ) )
- if self._hasouter:
- req.extend ( pack('>L', 1) )
- else:
- req.extend ( pack('>L', 0) )
-
- # token_filter
- req.extend ( pack('>L',len(self._tokenfilterlibrary)) + self._tokenfilterlibrary )
- req.extend ( pack('>L',len(self._tokenfiltername)) + self._tokenfiltername )
- req.extend ( pack('>L',len(self._tokenfilteropts)) + self._tokenfilteropts )
-
- # send query, get response
-
- self._reqs.append(req)
- return
-
-
- def RunQueries (self):
- """
- Run queries batch.
- Returns None on network IO failure; or an array of result set hashes on success.
- """
- if len(self._reqs)==0:
- self._error = 'no queries defined, issue AddQuery() first'
- return None
-
- sock = self._Connect()
- if not sock:
- return None
-
- req = bytearray()
- for r in self._reqs:
- req.extend(r)
- length = len(req)+8
- req_all = bytearray()
- req_all.extend(pack('>HHLLL', SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, length, 0, len(self._reqs)))
- req_all.extend(req)
- self._Send ( sock, req_all )
-
- response = self._GetResponse(sock, VER_COMMAND_SEARCH)
- if not response:
- return None
-
- nreqs = len(self._reqs)
-
- # parse response
- max_ = len(response)
- p = 0
-
- results = []
- for i in range(0,nreqs,1):
- result = {}
- results.append(result)
-
- result['error'] = ''
- result['warning'] = ''
- status = unpack('>L', response[p:p+4])[0]
- p += 4
- result['status'] = status
- if status != SEARCHD_OK:
- length = unpack('>L', response[p:p+4])[0]
- p += 4
- message = bytes_str(response[p:p+length])
- p += length
-
- if status == SEARCHD_WARNING:
- result['warning'] = message
- else:
- result['error'] = message
- continue
-
- # read schema
- fields = []
- attrs = []
-
- nfields = unpack('>L', response[p:p+4])[0]
- p += 4
- while nfields>0 and p<max_:
- nfields -= 1
- length = unpack('>L', response[p:p+4])[0]
- p += 4
- fields.append(bytes_str(response[p:p+length]))
- p += length
-
- result['fields'] = fields
-
- nattrs = unpack('>L', response[p:p+4])[0]
- p += 4
- while nattrs>0 and p<max_:
- nattrs -= 1
- length = unpack('>L', response[p:p+4])[0]
- p += 4
- attr = bytes_str(response[p:p+length])
- p += length
- type_ = unpack('>L', response[p:p+4])[0]
- p += 4
- attrs.append([attr,type_])
-
- result['attrs'] = attrs
-
- # read match count
- count = unpack('>L', response[p:p+4])[0]
- p += 4
- id64 = unpack('>L', response[p:p+4])[0]
- p += 4
-
- # read matches
- result['matches'] = []
- while count>0 and p<max_:
- count -= 1
- if id64:
- doc, weight = unpack('>QL', response[p:p+12])
- p += 12
- else:
- doc, weight = unpack('>2L', response[p:p+8])
- p += 8
-
- match = { 'id':doc, 'weight':weight, 'attrs':{} }
- for i in range(len(attrs)):
- if attrs[i][1] == SPH_ATTR_FLOAT:
- match['attrs'][attrs[i][0]] = unpack('>f', response[p:p+4])[0]
- elif attrs[i][1] == SPH_ATTR_BIGINT:
- match['attrs'][attrs[i][0]] = unpack('>q', response[p:p+8])[0]
- p += 4
- elif attrs[i][1] == SPH_ATTR_STRING:
- slen = unpack('>L', response[p:p+4])[0]
- p += 4
- match['attrs'][attrs[i][0]] = ''
- if slen>0:
- match['attrs'][attrs[i][0]] = bytes_str(response[p:p+slen])
- p += slen-4
- elif attrs[i][1] == SPH_ATTR_FACTORS:
- slen = unpack('>L', response[p:p+4])[0]
- p += 4
- match['attrs'][attrs[i][0]] = ''
- if slen>0:
- match['attrs'][attrs[i][0]] = response[p:p+slen-4]
- p += slen-4
- p -= 4
- elif attrs[i][1] == SPH_ATTR_MULTI:
- match['attrs'][attrs[i][0]] = []
- nvals = unpack('>L', response[p:p+4])[0]
- p += 4
- for n in range(0,nvals,1):
- match['attrs'][attrs[i][0]].append(unpack('>L', response[p:p+4])[0])
- p += 4
- p -= 4
- elif attrs[i][1] == SPH_ATTR_MULTI64:
- match['attrs'][attrs[i][0]] = []
- nvals = unpack('>L', response[p:p+4])[0]
- nvals = nvals/2
- p += 4
- for n in range(0,nvals,1):
- match['attrs'][attrs[i][0]].append(unpack('>q', response[p:p+8])[0])
- p += 8
- p -= 4
- else:
- match['attrs'][attrs[i][0]] = unpack('>L', response[p:p+4])[0]
- p += 4
-
- result['matches'].append ( match )
-
- result['total'], result['total_found'], result['time'], words = unpack('>4L', response[p:p+16])
-
- result['time'] = '%.3f' % (result['time']/1000.0)
- p += 16
-
- result['words'] = []
- while words>0:
- words -= 1
- length = unpack('>L', response[p:p+4])[0]
- p += 4
- word = bytes_str(response[p:p+length])
- p += length
- docs, hits = unpack('>2L', response[p:p+8])
- p += 8
-
- result['words'].append({'word':word, 'docs':docs, 'hits':hits})
-
- self._reqs = []
- return results
-
-
- def BuildExcerpts (self, docs, index, words, opts=None):
- """
- Connect to searchd server and generate exceprts from given documents.
- """
- if not opts:
- opts = {}
-
- assert(isinstance(docs, list))
- assert(isinstance(index, (str,text_type)))
- assert(isinstance(words, (str,text_type)))
- assert(isinstance(opts, dict))
-
- sock = self._Connect()
-
- if not sock:
- return None
-
- # fixup options
- opts.setdefault('before_match', '<b>')
- opts.setdefault('after_match', '</b>')
- opts.setdefault('chunk_separator', ' ... ')
- opts.setdefault('html_strip_mode', 'index')
- opts.setdefault('limit', 256)
- opts.setdefault('limit_passages', 0)
- opts.setdefault('limit_words', 0)
- opts.setdefault('around', 5)
- opts.setdefault('start_passage_id', 1)
- opts.setdefault('passage_boundary', 'none')
-
- # build request
- # v.1.0 req
-
- flags = 1 # (remove spaces)
- if opts.get('exact_phrase'): flags |= 2
- if opts.get('single_passage'): flags |= 4
- if opts.get('use_boundaries'): flags |= 8
- if opts.get('weight_order'): flags |= 16
- if opts.get('query_mode'): flags |= 32
- if opts.get('force_all_words'): flags |= 64
- if opts.get('load_files'): flags |= 128
- if opts.get('allow_empty'): flags |= 256
- if opts.get('emit_zones'): flags |= 512
- if opts.get('load_files_scattered'): flags |= 1024
-
- # mode=0, flags
- req = bytearray()
- req.extend(pack('>2L', 0, flags))
-
- # req index
- index = str_bytes(index)
- req.extend(pack('>L', len(index)))
- req.extend(index)
-
- # req words
- words = str_bytes(words)
- req.extend(pack('>L', len(words)))
- req.extend(words)
-
- # options
- opts_before_match = str_bytes(opts['before_match'])
- req.extend(pack('>L', len(opts_before_match)))
- req.extend(opts_before_match)
-
- opts_after_match = str_bytes(opts['after_match'])
- req.extend(pack('>L', len(opts_after_match)))
- req.extend(opts_after_match)
-
- opts_chunk_separator = str_bytes(opts['chunk_separator'])
- req.extend(pack('>L', len(opts_chunk_separator)))
- req.extend(opts_chunk_separator)
-
- req.extend(pack('>L', int(opts['limit'])))
- req.extend(pack('>L', int(opts['around'])))
-
- req.extend(pack('>L', int(opts['limit_passages'])))
- req.extend(pack('>L', int(opts['limit_words'])))
- req.extend(pack('>L', int(opts['start_passage_id'])))
- opts_html_strip_mode = str_bytes(opts['html_strip_mode'])
- req.extend(pack('>L', len(opts_html_strip_mode)))
- req.extend(opts_html_strip_mode)
- opts_passage_boundary = str_bytes(opts['passage_boundary'])
- req.extend(pack('>L', len(opts_passage_boundary)))
- req.extend(opts_passage_boundary)
-
- # documents
- req.extend(pack('>L', len(docs)))
- for doc in docs:
- doc = str_bytes(doc)
- req.extend(pack('>L', len(doc)))
- req.extend(doc)
-
- # send query, get response
- length = len(req)
-
- # add header
- req_head = bytearray()
- req_head.extend(pack('>2HL', SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, length))
- req_all = req_head + req
- self._Send ( sock, req_all )
-
- response = self._GetResponse(sock, VER_COMMAND_EXCERPT )
- if not response:
- return []
-
- # parse response
- pos = 0
- res = []
- rlen = len(response)
-
- for i in range(len(docs)):
- length = unpack('>L', response[pos:pos+4])[0]
- pos += 4
-
- if pos+length > rlen:
- self._error = 'incomplete reply'
- return []
-
- res.append(bytes_str(response[pos:pos+length]))
- pos += length
-
- return res
-
-
- def UpdateAttributes ( self, index, attrs, values, mva=False, ignorenonexistent=False ):
- """
- Update given attribute values on given documents in given indexes.
- Returns amount of updated documents (0 or more) on success, or -1 on failure.
-
- 'attrs' must be a list of strings.
- 'values' must be a dict with int key (document ID) and list of int values (new attribute values).
- optional boolean parameter 'mva' points that there is update of MVA attributes.
- In this case the 'values' must be a dict with int key (document ID) and list of lists of int values
- (new MVA attribute values).
- Optional boolean parameter 'ignorenonexistent' points that the update will silently ignore any warnings about
- trying to update a column which is not exists in current index schema.
-
- Example:
- res = cl.UpdateAttributes ( 'test1', [ 'group_id', 'date_added' ], { 2:[123,1000000000], 4:[456,1234567890] } )
- """
- assert ( isinstance ( index, str ) )
- assert ( isinstance ( attrs, list ) )
- assert ( isinstance ( values, dict ) )
- for attr in attrs:
- assert ( isinstance ( attr, str ) )
- for docid, entry in list(values.items()):
- AssertUInt32(docid)
- assert ( isinstance ( entry, list ) )
- assert ( len(attrs)==len(entry) )
- for val in entry:
- if mva:
- assert ( isinstance ( val, list ) )
- for vals in val:
- AssertInt32(vals)
- else:
- AssertInt32(val)
-
- # build request
- req = bytearray()
- index = str_bytes(index)
- req.extend( pack('>L',len(index)) + index )
-
- req.extend ( pack('>L',len(attrs)) )
- ignore_absent = 0
- if ignorenonexistent: ignore_absent = 1
- req.extend ( pack('>L', ignore_absent ) )
- mva_attr = 0
- if mva: mva_attr = 1
- for attr in attrs:
- attr = str_bytes(attr)
- req.extend ( pack('>L',len(attr)) + attr )
- req.extend ( pack('>L', mva_attr ) )
-
- req.extend ( pack('>L',len(values)) )
- for docid, entry in list(values.items()):
- req.extend ( pack('>Q',docid) )
- for val in entry:
- val_len = val
- if mva: val_len = len ( val )
- req.extend ( pack('>L',val_len ) )
- if mva:
- for vals in val:
- req.extend ( pack ('>L',vals) )
-
- # connect, send query, get response
- sock = self._Connect()
- if not sock:
- return None
-
- length = len(req)
- req_all = bytearray()
- req_all.extend( pack ( '>2HL', SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, length ) )
- req_all.extend( req )
- self._Send ( sock, req_all )
-
- response = self._GetResponse ( sock, VER_COMMAND_UPDATE )
- if not response:
- return -1
-
- # parse response
- updated = unpack ( '>L', response[0:4] )[0]
- return updated
-
-
- def BuildKeywords ( self, query, index, hits ):
- """
- Connect to searchd server, and generate keywords list for a given query.
- Returns None on failure, or a list of keywords on success.
- """
- assert ( isinstance ( query, str ) )
- assert ( isinstance ( index, str ) )
- assert ( isinstance ( hits, int ) )
-
- # build request
- req = bytearray()
- query = str_bytes(query)
- req.extend(pack ( '>L', len(query) ) + query)
- index = str_bytes(index)
- req.extend ( pack ( '>L', len(index) ) + index )
- req.extend ( pack ( '>L', hits ) )
-
- # connect, send query, get response
- sock = self._Connect()
- if not sock:
- return None
-
- length = len(req)
- req_all = bytearray()
- req_all.extend(pack ( '>2HL', SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, length ))
- req_all.extend(req)
- self._Send ( sock, req_all )
-
- response = self._GetResponse ( sock, VER_COMMAND_KEYWORDS )
- if not response:
- return None
-
- # parse response
- res = []
-
- nwords = unpack ( '>L', response[0:4] )[0]
- p = 4
- max_ = len(response)
-
- while nwords>0 and p<max_:
- nwords -= 1
-
- length = unpack ( '>L', response[p:p+4] )[0]
- p += 4
- tokenized = response[p:p+length]
- p += length
-
- length = unpack ( '>L', response[p:p+4] )[0]
- p += 4
- normalized = response[p:p+length]
- p += length
-
- entry = { 'tokenized':bytes_str(tokenized), 'normalized':bytes_str(normalized) }
- if hits:
- entry['docs'], entry['hits'] = unpack ( '>2L', response[p:p+8] )
- p += 8
-
- res.append ( entry )
-
- if nwords>0 or p>max_:
- self._error = 'incomplete reply'
- return None
-
- return res
-
- def Status ( self, session=False ):
- """
- Get the status
- """
-
- # connect, send query, get response
- sock = self._Connect()
- if not sock:
- return None
-
- sess = 1
- if session:
- sess = 0
-
- req = pack ( '>2HLL', SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, sess )
- self._Send ( sock, req )
-
- response = self._GetResponse ( sock, VER_COMMAND_STATUS )
- if not response:
- return None
-
- # parse response
- res = []
-
- p = 8
- max_ = len(response)
-
- while p<max_:
- length = unpack ( '>L', response[p:p+4] )[0]
- k = response[p+4:p+length+4]
- p += 4+length
- length = unpack ( '>L', response[p:p+4] )[0]
- v = response[p+4:p+length+4]
- p += 4+length
- res += [[bytes_str(k), bytes_str(v)]]
-
- return res
-
- ### persistent connections
-
- def Open(self):
- if self._socket:
- self._error = 'already connected'
- return None
-
- server = self._Connect()
- if not server:
- return None
-
- # command, command version = 0, body length = 4, body = 1
- request = pack ( '>hhII', SEARCHD_COMMAND_PERSIST, 0, 4, 1 )
- self._Send ( server, request )
-
- self._socket = server
- return True
-
- def Close(self):
- if not self._socket:
- self._error = 'not connected'
- return
- self._socket.close()
- self._socket = None
-
- def EscapeString(self, string):
- return re.sub(r"([=\(\)|\-!@~\"&/\\\^\$\=\<])", r"\\\1", string)
-
-
- def FlushAttributes(self):
- sock = self._Connect()
- if not sock:
- return -1
-
- request = pack ( '>hhI', SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ) # cmd, ver, bodylen
- self._Send ( sock, request )
-
- response = self._GetResponse ( sock, VER_COMMAND_FLUSHATTRS )
- if not response or len(response)!=4:
- self._error = 'unexpected response length'
- return -1
-
- tag = unpack ( '>L', response[0:4] )[0]
- return tag
-
-def AssertInt32 ( value ):
- assert(isinstance(value, (int, long)))
- assert(value>=-2**32-1 and value<=2**32-1)
-
-def AssertUInt32 ( value ):
- assert(isinstance(value, (int, long)))
- assert(value>=0 and value<=2**32-1)
-
-def SetBit ( flag, bit, on ):
- if on:
- flag += ( 1<<bit )
- else:
- reset = 255 ^ ( 1<<bit )
- flag = flag & reset
-
- return flag
-
-if sys.version_info > (3,):
- def str_bytes(x):
- return bytearray(x, 'utf-8')
-else:
- def str_bytes(x):
- if isinstance(x,unicode):
- return bytearray(x, 'utf-8')
- else:
- return bytearray(x)
-
-def bytes_str(x):
- assert (isinstance(x, bytearray))
- return x.decode('utf-8')
-
-#
-# $Id$
-#
diff --git a/sphinx_search.conf b/sphinx_search.conf
index 7740000..83947b1 100644
--- a/sphinx_search.conf
+++ b/sphinx_search.conf
@@ -45,10 +45,7 @@ indexer {
}
searchd {
- # Use this if you want to use python api
- listen = 9312
- # Use this if you want to http api
- #listen = localhost:8080:http
+ listen = 0.0.0.0:8080:http
query_log = /tmp/query.log
binlog_path = /tmp/
read_timeout = 5