Merge pull request #342 from groleo/regexp-search.py

sqlite3: add regexp searches to search.py

1 files changed, 255 insertions, 228 deletions

diff --git a/addon/doxypysql/search.py b/addon/doxypysql/search.py
index d0c88c0..c185138 100755
--- a/addon/doxypysql/search.py
+++ b/addon/doxypysql/search.py
@@ -14,6 +14,7 @@ import sys
 import os
 import getopt
 import json
+import re
 
 class MemberType:
   Define="0"
@@ -28,6 +29,7 @@ class MemberType:
   DCOP="9"
   Property="10"
   Event="11"
+  File="12"
 
 class RequestType:
   References="9901"
@@ -38,252 +40,253 @@ class RequestType:
   BaseClasses="9906"
   SubClasses="9907"
 
-g_conn=None
+g_use_regexp=False
 ###############################################################################
 
-def escapeLike(val):
-    return 'LIKE "%' + val.replace("\\", "\\\\").replace("_", "\\_") \
-        .replace("%", "\\%") + '%" ESCAPE "\\"'
+# case-insensitive sqlite regexp function
+def re_fn(expr, item):
+    reg = re.compile(expr, re.I)
+    return reg.search(item) is not None
 
-def matchName(name):
-    if type(name) is str:
-        return "name "+escapeLike(name)
-    else:
-        return 'id=%d' %name
-
-def fileName(id_file):
-    if g_conn.execute("SELECT COUNT(*) FROM files WHERE id=?",[id_file]).fetchone()[0] > 1:
-        print "non-uniq fileid"
-
-    for r in g_conn.execute("SELECT * FROM files WHERE id=?",[id_file]).fetchall():
-            return r['name']
+def openDb(dbname):
+    if dbname == None:
+        dbname = "doxygen_sqlite3.db"
 
-    return ""
+    if not os.path.isfile(dbname):
+        raise BaseException("No such file %s" % dbname )
 
-def fileId(name):
-    if g_conn.execute("SELECT COUNT(*) FROM files WHERE name=?",[name]).fetchone()[0] > 1:
-        print "non-uniq file name"
+    conn = sqlite3.connect(dbname)
+    conn.execute('PRAGMA temp_store = MEMORY;')
+    conn.row_factory = sqlite3.Row
+    conn.create_function("REGEXP", 2, re_fn)
+    return conn
+###############################################################################
+class Finder:
+    def __init__(self,cn,name,row_type=str):
+        self.cn=cn
+        self.name=name
+        self.row_type=row_type
+
+    def match(self,row):
+        if self.row_type is int:
+            return " id=?"
+        else:
+            if g_use_regexp == True:
+                return " REGEXP (?,%s)" %row
+            else:
+                return " %s=?" %row
 
-    for r in g_conn.execute("SELECT * FROM files WHERE name=?",[name]).fetchall():
-            return r['id']
+    def fileName(self,id_file):
+        if self.cn.execute("SELECT COUNT(*) FROM files WHERE rowid=?",[id_file]).fetchone()[0] > 1:
+            print >>sys.stderr,"WARNING: non-uniq fileid [%s]. Considering only the first match." % id_file
 
-    return -1
+        for r in self.cn.execute("SELECT * FROM files WHERE rowid=?",[id_file]).fetchall():
+                return r['name']
 
-###############################################################################
+        return ""
 
-def findReferences(name):
-    o=[]
+    def fileId(self,name):
+        if self.cn.execute("SELECT COUNT(*) FROM files WHERE"+self.match("name"),[name]).fetchone()[0] > 1:
+            print >>sys.stderr,"WARNING: non-uniq file name [%s]. Considering only the first match." % name
 
-    cur = g_conn.cursor()
-    cur.execute("SELECT refid FROM memberdef WHERE name=?",[name])
-    refids = cur.fetchall()
+        for r in self.cn.execute("SELECT rowid FROM files WHERE"+self.match("name"),[name]).fetchall():
+                return r[0]
 
-    if len(refids) == 0:
+        return -1
+###############################################################################
+    def references(self):
+        o=[]
+        cur = self.cn.cursor()
+        cur.execute("SELECT refid FROM memberdef WHERE"+self.match("name"),[self.name])
+        refids = cur.fetchall()
+
+        if len(refids) == 0:
+            return o
+
+        refid = refids[0]['refid']
+        cur = self.cn.cursor()
+        #TODO:SELECT rowid from refids where refid=refid
+        for info in cur.execute("SELECT * FROM xrefs WHERE refid_dst LIKE '%"+refid+"%'"):
+            item={}
+            cur = self.cn.cursor()
+            for i2 in cur.execute("SELECT * FROM memberdef WHERE refid=?",[info['src']]):
+                item['name']=i2['name']
+                item['src']=info['src']
+                item['file']=self.fileName(info['id_file'])
+                item['line']=info['line']
+
+            o.append(item)
         return o
-
-    refid = refids[0]['refid']
-    cur = g_conn.cursor()
-    for info in cur.execute("SELECT * FROM xrefs WHERE dst LIKE '%"+refid+"%'"):
-        item={}
-        cur = g_conn.cursor()
-        for i2 in cur.execute("SELECT * FROM memberdef WHERE refid=?",[info['src']]):
-            item['name']=i2['name']
-            item['src']=info['src']
-            item['file']=fileName(info['id_file'])
-            item['line']=info['line']
-
-        o.append(item)
-    return o
-
-
-def findFunction(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM memberdef WHERE '+matchName(name)+' AND kind=?',[MemberType.Function]).fetchall():
-        item={}
-        item['name'] = r['name']
-        item['definition'] = r['definition']
-        item['argsstring'] = r['argsstring']
-        item['file'] = fileName(r['id_file'])
-        item['line'] = r['line']
-        item['detaileddescription'] = r['detaileddescription']
-        o.append(item)
-    return o
-
-
-def findMacro(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM memberdef WHERE '+matchName(name)+' AND kind=?',[MemberType.Define]).fetchall():
-        item={}
-        item['name'] = r['name']
-        if r['argsstring']:
+###############################################################################
+    def function(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM memberdef WHERE'+self.match("name")+' AND kind=?',[self.name,MemberType.Function])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            item['definition'] = r['definition']
             item['argsstring'] = r['argsstring']
-        item['definition'] = r['initializer']
-        item['file'] = fileName(r['id_file'])
-        item['line'] = r['line']
-        o.append(item)
-    return o
-
-
-def findTypedef(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM memberdef WHERE '+matchName(name)+' AND kind=?',[MemberType.Typedef]).fetchall():
-        item={}
-        item['name'] = r['name']
-        item['definition'] = r['definition']
-        item['file'] = fileName(r['id_file'])
-        item['line'] = r['line']
-        o.append(item)
-    return o
-
-
-def findVariable(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM memberdef WHERE '+matchName(name)+' AND kind=?',[MemberType.Variable]).fetchall():
-        item={}
-        item['name'] = r['name']
-        item['definition'] = r['definition']
-        item['file'] = fileName(r['id_file'])
-        item['line'] = r['line']
-        o.append(item)
-    return o
-
-def findParams(name):
-    o=[]
-    for r in g_conn.execute('SELECT id FROM memberdef WHERE '+matchName(name)).fetchall():
-        #a=("SELECT * FROM params where id=(SELECT id_param FROM memberdef_params where id_memberdef=?",[id_memberdef])
-        item={}
-        item['id'] = r['id']
-        o.append(item)
-    return o
-
-
-def findStruct(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM compounddef WHERE '+matchName(name)).fetchall():
-        item={}
-        item['name'] = r['name']
-        o.append(item)
-    return o
-
-def findIncluders(name):
-    o=[]
-    fid = fileId(name)
-    for r in g_conn.execute('SELECT * FROM includes WHERE id_dst=?',[fid]).fetchall():
-        item={}
-        item['name'] = fileName(r['id_src'])
-        o.append(item)
-    return o
-
-def findIncludees(name):
-    o=[]
-    fid = fileId(name)
-    for r in g_conn.execute('SELECT * FROM includes WHERE id_src=?',[fid]).fetchall():
-        item={}
-        item['name'] = r['dst']
-        o.append(item)
-    return o
-
-
-def findMembers(name):
-    o=[]
-    for r in g_conn.execute('SELECT * FROM memberdef WHERE scope LIKE "%'+name+'%";').fetchall():
-        item={}
-        item['name'] = r['name']
-        item['definition'] = r['definition']
-        item['argsstring'] = r['argsstring']
-        item['file'] = fileName(r['id_file'])
-        item['line'] = r['line']
-        item['documentation'] = r['documentation']
-        o.append(item)
-    return o
-
-
-def findBaseClasses(name):
-    o=[]
-    for r in g_conn.execute('SELECT base FROM basecompoundref WHERE derived=?',[name]).fetchall():
-        item={}
-        item['name'] = r['base']
-        o.append(item)
-    return o
-
-
-def findSubClasses(name):
-    o=[]
-    for r in g_conn.execute('SELECT derived FROM basecompoundref WHERE base=?',[name]).fetchall():
-        item={}
-        item['name'] = r['derived']
-        o.append(item)
-    return o
-
-
+            item['file'] = self.fileName(r['id_file'])
+            item['line'] = r['line']
+            item['detaileddescription'] = r['detaileddescription']
+            o.append(item)
+        return o
 ###############################################################################
+    def file(self):
+        o=[]
+        for r in self.cn.execute("SELECT rowid,* FROM files WHERE"+self.match("name"),[self.name]).fetchall():
+            item={}
+            item['name'] = r['name']
+            item['id'] =   r['rowid']
+            o.append(item)
+        return o
 
-def usage():
-  print """Usage: search.py [Options]
-Options:
-    -h, --help
-    -d <D>    Use database <D> for queries
-    -r <F>    Search for references to <F>
-    -f <F>    Search for definition of function <F>
-    -m <M>    Search for definition of macro <M>
-    -t <T>    Search for definition of type <T>
-    -v <V>    Search for definition of variable <V>
-    -I <I>    Get the includers of <I>
-    -i <I>    Get the includees of <I>
-    -M <C>    Get all members of class <C>
-    -B <C>    Get the base classes of class <C>
-    -S <C>    Get the sub classes of class <C>
-"""
-
-def openDb(dbname):
-    global g_conn
-
-    if dbname == None:
-        dbname = "doxygen_sqlite3.db"
-
-    if not os.path.isfile(dbname):
-        raise BaseException("No such file %s" % dbname )
-
-    g_conn = sqlite3.connect(dbname)
-    g_conn.execute('PRAGMA temp_store = MEMORY;')
-    g_conn.row_factory = sqlite3.Row
-
 ###############################################################################
-def process(kind,o):
+    def macro(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM memberdef WHERE'+self.match("name")+' AND kind=?',[self.name,MemberType.Define])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            if r['argsstring']:
+                item['argsstring'] = r['argsstring']
+            item['definition'] = r['initializer']
+            item['file'] = self.fileName(r['id_file'])
+            item['line'] = r['line']
+            o.append(item)
+        return o
+###############################################################################
+    def typedef(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM memberdef WHERE'+self.match("name")+' AND kind=?',[self.name,MemberType.Typedef])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            item['definition'] = r['definition']
+            item['file'] = self.fileName(r['id_file'])
+            item['line'] = r['line']
+            o.append(item)
+        return o
+###############################################################################
+    def variable(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM memberdef WHERE'+self.match("name")+' AND kind=?',[self.name,MemberType.Variable])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            item['definition'] = r['definition']
+            item['file'] = self.fileName(r['id_file'])
+            item['line'] = r['line']
+            o.append(item)
+        return o
+###############################################################################
+    def params(self):
+        o=[]
+        c=self.cn.execute('SELECT id FROM memberdef WHERE'+self.match("name"),[self.name])
+        for r in c.fetchall():
+            #a=("SELECT * FROM params where id=(SELECT id_param FROM memberdef_params where id_memberdef=?",[id_memberdef])
+            item={}
+            item['id'] = r['id']
+            o.append(item)
+        return o
+###############################################################################
+    def struct(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM compounddef WHERE'+self.match("name"),[self.name])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            o.append(item)
+        return o
+###############################################################################
+    def includers(self):
+        o=[]
+        fid = self.fileId(self.name)
+        c=self.cn.execute('SELECT * FROM includes WHERE id_dst=?',[fid])
+        for r in c.fetchall():
+            item={}
+            item['name'] = self.fileName(r['id_src'])
+            o.append(item)
+        return o
+###############################################################################
+    def includees(self):
+        o=[]
+        fid = self.fileId(self.name)
+        c=self.cn.execute('SELECT * FROM includes WHERE id_src=?',[fid])
+        for r in c.fetchall():
+            item={}
+            item['name'] = self.fileName(r['id_dst'])
+            o.append(item)
+        return o
+###############################################################################
+    def members(self):
+        o=[]
+        c=self.cn.execute('SELECT * FROM memberdef WHERE'+self.match("scope"),[self.name])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['name']
+            item['definition'] = r['definition']
+            item['argsstring'] = r['argsstring']
+            item['file'] = self.fileName(r['id_file'])
+            item['line'] = r['line']
+            #item['documentation'] = r['documentation']
+            o.append(item)
+        return o
+###############################################################################
+    def baseClasses(self):
+        o=[]
+        c=self.cn.execute('SELECT base FROM basecompoundref WHERE'+self.match("derived"),[self.name])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['base']
+            o.append(item)
+        return o
+###############################################################################
+    def subClasses(self):
+        o=[]
+        c=self.cn.execute('SELECT derived FROM basecompoundref WHERE'+self.match("base"),[self.name])
+        for r in c.fetchall():
+            item={}
+            item['name'] = r['derived']
+            o.append(item)
+        return o
+###############################################################################
+def process(f,kind):
     request_processors = {
-        MemberType.Function: findFunction,
-        MemberType.Define:   findMacro,
-        MemberType.Variable: findVariable,
-        MemberType.Typedef:  findTypedef,
-        RequestType.References: findReferences,
-        RequestType.Struct: findStruct,
-        RequestType.Includers: findIncluders,
-        RequestType.Includees: findIncludees,
-        RequestType.Members: findMembers,
-        RequestType.BaseClasses: findBaseClasses,
-        RequestType.SubClasses: findSubClasses
+        MemberType.Function: f.function,
+        MemberType.File: f.file,
+        MemberType.Define:   f.macro,
+        MemberType.Variable: f.variable,
+        MemberType.Typedef:  f.typedef,
+        RequestType.References: f.references,
+        RequestType.Struct: f.struct,
+        RequestType.Includers: f.includers,
+        RequestType.Includees: f.includees,
+        RequestType.Members: f.members,
+        RequestType.BaseClasses: f.baseClasses,
+        RequestType.SubClasses: f.subClasses
     }
-    return request_processors[kind](o)
-
-
-def processHref(ref):
+    return request_processors[kind]()
+###############################################################################
+def processHref(cn,ref):
     j={}
 
     # is it in memberdef ?
     table="memberdef"
-    if ( g_conn.execute("SELECT count(*) from %s WHERE refid='%s'" % (table,ref) ).fetchone()[0] > 0 ):
-        for r in g_conn.execute("SELECT kind,id FROM %s WHERE refid='%s'" % (table,ref) ).fetchall():
-            j=process(str(r['kind']),int(r['id']))
+    if ( cn.execute("SELECT count(*) from %s WHERE refid=?"%table,[ref] ).fetchone()[0] > 0 ):
+        for r in cn.execute("SELECT kind,id FROM %s WHERE refid='%s'" % (table,ref) ).fetchall():
+            f=Finder(cn,r['id'],int)
+            j=process(f,str(r['kind']))
 
     # is it in compounddef ?
     table="compounddef"
-    if ( g_conn.execute("SELECT count(*) from %s WHERE refid='%s'" % (table,ref)).fetchone()[0] > 0 ):
-        for r in g_conn.execute("SELECT id FROM %s WHERE refid='%s'" % (table,ref) ).fetchall():
-            j=process(RequestType.Struct,int(r['id']))
+    if ( cn.execute("SELECT count(*) from %s WHERE refid=?"%table,[ref]).fetchone()[0] > 0 ):
+        for r in cn.execute("SELECT id FROM %s WHERE refid=?"%table,[ref] ).fetchall():
+            f=Finder(cn,r['id'],int)
+            j=process(f,RequestType.Struct)
 
     return j
-
-
+###############################################################################
 def serveCgi():
     import cgi
 
@@ -298,16 +301,33 @@ def serveCgi():
         print '{"result": null, "error": "no refid given"}'
         sys.exit(0)
 
-    openDb('doxygen_sqlite3.db')
+    cn=openDb('doxygen_sqlite3.db')
 
-    j = processHref(ref)
+    j = processHref(cn,ref)
 
     print json.dumps({"result":j,"error":None})
-
-
+###############################################################################
+def usage():
+  print >>sys.stderr,"""Usage: search.py [Options]
+Options:
+    -h, --help
+    -d <D>    Use database <D> for queries.
+    -f <F>    Search for definition of function <F>.
+    -m <M>    Search for definition of macro <M>.
+    -r <F>    Search for references to function <F>.
+    -t <T>    Search for definition of type <T>.
+    -v <V>    Search for definition of variable <V>.
+    -I <I>    What files are including <I>.
+    -i <i>    What files are included by <i>.
+    -B <C>    Get the base classes of class <C>.
+    -M <C>    Get all members of class <C>.
+    -S <C>    Get the sub classes of class <C>.
+    -R        Consider the search <term> to be a regex.
+"""
+###############################################################################
 def serveCli(argv):
     try:
-        opts, args = getopt.getopt(argv, "hr:I:i:d:f:m:t:v:H:M:B:S:",["help"])
+        opts, args = getopt.getopt(argv, "hr:RI:i:d:f:m:t:v:H:M:B:S:F:",["help"])
     except getopt.GetoptError:
         usage()
         sys.exit(1)
@@ -315,6 +335,7 @@ def serveCli(argv):
     ref=None
     dbname=None
     j={}
+    global g_use_regexp
 
     for a, o in opts:
         if a in ('-h', '--help'):
@@ -325,6 +346,9 @@ def serveCli(argv):
             continue
         elif a in ('-r'):
             kind=RequestType.References
+        elif a in ('-R'):
+            g_use_regexp=True
+            continue
         elif a in ('-I'):
             kind=RequestType.Includers
         elif a in ('-i'):
@@ -337,6 +361,8 @@ def serveCli(argv):
             kind=RequestType.SubClasses
         elif a in ('-f'):
             kind=MemberType.Function
+        elif a in ('-F'):
+            kind=MemberType.File
         elif a in ('-m'):
             kind=MemberType.Define
         elif a in ('-t'):
@@ -346,12 +372,13 @@ def serveCli(argv):
         elif a in ('-H'):
             ref = o
 
-        openDb(dbname)
+        cn=openDb(dbname)
+        f=Finder(cn,o)
         if ref != None:
-          j=processHref(ref)
+          j=processHref(cn,ref)
         else:
-          j=process(kind,o)
-        print json.dumps(j)
+          j=process(f,kind)
+        print json.dumps(j,indent=4)
 
 
 def main(argv):