Index: Document.py
===================================================================
--- Document.py	(revision 111)
+++ Document.py	(working copy)
@@ -2,9 +2,11 @@
 # Copyright 2004-2006 by Vahur Rebas
 
 import re
+import logging
 import Globals
 from Globals import Acquisition, Persistent
 from AccessControl import ClassSecurityInfo
+from AccessControl import getSecurityManager
 from Acquisition import aq_base, aq_inner, aq_parent, Implicit
 from OFS.SimpleItem import SimpleItem
 from OFS.PropertyManager import PropertyManager
@@ -26,7 +28,10 @@
 from interfaces import IDocument
 from permissions import *
 from schemas import get_schema
+from utils import invokeScript
 
+logger = logging.getLogger('Korpus')
+
 class Document(Persistent,SimpleItem,CatalogAware, PropertyManager):
     """ One document """
     
@@ -72,10 +77,9 @@
         self._document_schema = _schema
         self._corpus = ''
         self._dsstore = {}
-        self._lemmated = False
-        self._parsed = False
+        self.textcontent = ""
 
-        self._analysis = ''
+        self._tmp_doc = True
 
     def __getattr__(self, name):
         if name in get_schema(self.getMetaSchema()).getFieldNames():
@@ -137,6 +141,11 @@
 
     def isDeleted(self):
         return self.getProperty('is_deleted')
+
+    security.declareProtected(perm_del_documents, 'deleteMe')
+    def deleteMe(self):
+        self._updateProperty('is_deleted', 1)
+        self.sqls.docDeleted(docid=self.getId())
         
     security.declareProtected(perm_view_document, 'getBody')
     def getBody(self):
@@ -163,6 +172,9 @@
         self.REQUEST.RESPONSE.setHeader("Content-type","text/html; charset=UTF-8")
         return self.fulldoc
 
+    def getVisibleText(self):
+        return self.textcontent
+
     security.declareProtected(perm_view_document, 'getStatus')
     def getStatus(self):
         """ return document status 
@@ -191,23 +203,15 @@
         """
         if not self.getStatus():
             self._updateProperty('document_status', 1)
+            self.sqls.docStatus(docid=self.getId(), status=1)
             import copy
             self.org_fulldoc = copy.deepcopy(self.fulldoc)
             self.org_title = copy.deepcopy(self.title)
             self.org_description = copy.deepcopy(self.description)
             self.org_body = copy.deepcopy(self.body)
             self.reindex_object()
-        return REQUEST.RESPONSE.redirect('document_mark')
+        return REQUEST.RESPONSE.redirect('mark.html')
 
-    def isLemmated(self):
-        return getattr(self, '_lemmated', False)
-
-    def isParsed(self):
-        return getattr(self, '_parsed', False)
-
-    def getAnalysis(self):
-        return getattr(self, '_analysis', "")
-    
     security.declareProtected(perm_edit_document, 'extractTitle')
     def extractTitle(self):
         """ exract title from Document """
@@ -231,9 +235,13 @@
         doc = xml.dom.minidom.parseString(self.getDocument())
         bodytag = doc.getElementsByTagName('body')[0]
         stri = bodytag.toxml()
+        print stri
         b = re.compile('(<body>)|(</body>)', re.I)
         stri = stri.encode('utf-8')
         res = b.sub('', stri)
+        print
+        print "RES:"
+        print res
         res = unicode(res, 'utf-8').encode('utf-8')
         return res
     
@@ -356,9 +364,6 @@
             if start.startswith('/'): start = start[1:]
             if end.startswith('/'): end = end[1:]
             
-            print self.getDocument()
-            print body
-            print start
             try:
                 start_node = xpath.Evaluate(start, body)[0] # always take the
             except IndexError, ie:
@@ -375,12 +380,10 @@
             author = str(REQUEST.AUTHENTICATED_USER)
             if type(content) == types.UnicodeType:
                 content = content.encode('utf-8')
-            err = self.Errors.addNewError(pointer, content, self.getId(), code, author)
             pre, post = self.extractContext(body, start_node, end_node, int(st_offset), int(en_offset), start=1)
             pre.reverse()
-            err.addPreContext(pre)
-            err.addPostContext(post)
-                                            
+            err = self.Errors.addNewError(pointer, content, self.getId(), code, author, pre, post)
+
         self.reindex_object()
         return REQUEST.RESPONSE.redirect(self.absolute_url())
 
@@ -495,18 +498,129 @@
                 record = res[1]
         return [result, record]
 
+    def getTags(self):
+        """ get usable tags """
+        u = str(getSecurityManager().getUser())
+        tagresults = self.sqls.tagsGet(uname=u)
+        doctagsresults = self.sqls.tagGetByDocument(docid=self.getId())
+        mytags = [ t['id'] for t in tagresults ]
+        doctags = [t['tag'] for t in doctagsresults if t['tag'] in mytags ]
+        res = []
+        for t in tagresults:
+            r = [t['id'], t['label'], None]
+            r[2] = t['id'] in doctags
+            res.append(r)
+        return res
+
+    def setTagJS(self, tagid):
+        """ set tag """
+        u = str(getSecurityManager().getUser())
+        t = self.sqls.tagGet(tagid=tagid)
+        if len(t) != 1:
+            raise 'oops. tag error!'
+        tagline = t[0]
+        invokeScript(tagline['script'], [self.getId()])
+        self.sqls.tagDocument(uname=u, docid=self.getId(), tagid=tagid)
+        return "1"
+
+    def removeTagJS(self, tagid):
+        """ remove tag """
+        u = str(getSecurityManager().getUser())
+        self.sqls.tagDocumentRemove(uname=u, tagid=tagid)
+        return "1"
+
     security.declareProtected(perm_mark_document, 'useThis')
     def useThis(self):
         """ user will start to use this document for his own personal study """
         return 1
 
+    def getSentences(self):
+        ss = self.toText(self.getBody()).split('\n')
+        return [ s for s in ss if s ]
+
     security.declareProtected(perm_edit_document, 'toText')
-    def toText(self):
+    def toText(self, htm=None):
         """ convert html string to text """
-        C = html.Converter()
-        a = C.convert(unicode(self.getDocument(),'utf-8'), 'utf-8')
-        return a[0]
+        if htm is None:
+            htm = self.getDocument()
+        # C = html.Converter()
+        # a = C.convert(unicode(htm, 'utf-8'), 'utf-8')
+        # return a[0]
+        txt = re.sub('<br.*?>', '\n', htm)
+        txt = txt.replace('\r', '')
+        txt = re.sub('<.*?>', '', txt)
+        #txt = txt.replace('\n', ' ')
+        txt = re.sub('(?<![.!?])\n', ' ', txt)
+        txt = txt.replace('\xe2\x80\x9e', '"')
+        txt = txt.replace('\xe2\x80\x9c', '"')
+        txt = txt.replace('\xe2\x80\x9d', '"')
+        txt = txt.replace('\xe2\x80\xa2', '"')
+        txt = txt.replace('\xe2\x80\xa2', '*')
+        txt = txt.replace('\xe2\x80\x93', '-')
+        txt = txt.replace('\xe2\x80\xa6', '...')
+        def repl(g):
+            return g.group(0)+"\n"
+        txt = re.sub('[!?.]', repl, txt)
+        p = re.compile('^ +', re.M)
+        txt = p.sub('', txt)
+        txt = txt.decode('utf-8')
+        txt = self._fix_dates(txt)
+        txt = self._fix_names(txt)
+        txt = self._fix_years(txt)
+        txt = self._fix_shors(txt)
+        txt = self._fix_dots(txt)
+        sentences = [ x.strip() for x in txt.split('\n') if x.strip() ]
+        #for x in sentences:
+        #    print "*", x
+        return txt
 
+    def _fix_dates(self, intxt):
+        """ fixes 29. veebruar like cases """
+        p = re.compile('(?P<num>[0-9]+\.)([ \n]+)(?P<mon>(jaan|veeb|mär|apr|mai|juuni|juuli|august|septem|oktoob|nov|det|saj|eluaast))', re.I|re.S)
+        res, subcount = p.subn(r'\g<num> \g<mon>', intxt)
+        if subcount > 0:
+            print "date fix for %s" % self.getId(), subcount
+        return res
+
+    def _fix_names(self, res):
+        """ fixes A. Kivikas """
+        pats = [re.compile('(?P<pref>( |\n))(?P<char>.\.)\n+', re.I|re.S),
+            re.compile('(?P<pref>( |\n))(?P<char>.\.) \n+', re.I|re.S)]
+        for p in pats:
+            for x in range(2):
+                subcount = -1
+                res, subcount = p.subn(r'\g<pref>\g<char> ', res)
+                if subcount > 0: print "name fix r%s for %s" % ( x, self.getId()), subcount
+                else: print "."
+        return res
+    
+    def _fix_shors(self, res):
+        """ shorts like dr. """
+        pats = [re.compile(' (?P<short>(dr|pr|mr)\.)\n', re.I|re.S)]
+        for p in pats:
+            res, subcount = p.subn(r' \g<short> ', res)
+        return res
+
+    def _fix_dots(self, res):
+        """ ... """
+        for x in range(1,20):
+            res = res.replace('.\n.', '..')
+        res = res.replace('...\n', '... ')
+        res = res.replace('... \n', '... ')
+        res = re.sub('(?P<left>\(.*?)\n(?P<right>.*?\))', r'\g<left>\g<right>', res)
+        return res
+
+    def _fix_years(self, intxt):
+        p1 = re.compile('(?P<year>[0-9]+\.)([\n ])+(?P<aft>(aasta|a\.))', re.I|re.S)
+        p2 = re.compile('(?P<left>[0-9]+\.)[ \n]+(?P<right>-[0-9]+)', re.I|re.S)
+        res, subcount = p1.subn(r'\g<year> \g<aft>', intxt)
+        if subcount > 0:
+            print "year fix for %s" % self.getId(), subcount
+        res, subcount = p2.subn(r'\g<left>\g<right>', res)
+        if subcount > 0:
+            print "year range fix for %s" % self.getId(), subcount
+        return res
+
     security.declareProtected(perm_view_document, 'get_n_of_errors')
     def get_n_of_errors(self):
         codes = self.Errors.getDocumentMarks(self.getId())
@@ -828,6 +942,11 @@
         #    print txt
         #    print "="*40
         #    print value
+    def getMorfoView(self):
+        x = self.sqls.morfoRaw(docid=self.getId())
+        if len(x) == 0:
+            return "Not available"
+        return x[0][0]
 
     def _updateDocStatistics(self):
         """ doctest """
@@ -852,12 +971,24 @@
         self.sqls.storeDocument(
                 docid=self.getId(), title=self.getTitle(), 
                 language=getattr(self, 'docLanguage', 'x'), corpus=self.getCorpus())
+        # self.sqls.storeDocContent(
+        #         docid = self.getId(),
+        #         html = self.getBody(),
+        #         plaintext = self.toText(self.getBody()))
+        self.sqls.storeDocContent(
+                docid = self.getId(),
+                html = "",
+                plaintext = self.toText(self.getBody()))
+        # ss = self.getSentences()
+        # self.sqls.deleteSentencesByDocument(docid=self.getId())
+        # for s in ss:
+        #     self.sqls.addSentence(docid = self.getId(), lang=getattr(self, 'docLanguage', 'x'), sentence=s)
       
 def get_text(elem):
-	""" used only to extract documents title... """
-	res=u''
-	for line in elem.childNodes:
-		res += line.nodeValue
-	return res.encode('utf-8')
+    """ used only to extract documents title... """
+    res=u''
+    for line in elem.childNodes:
+        res += line.nodeValue
+    return res.encode('utf-8')
 
 Globals.InitializeClass(Document)
