1   /*
2    *  DatabaseDocumentImpl.java
3    *
4    *  Copyright (c) 1998-2001, The University of Sheffield.
5    *
6    *  This file is part of GATE (see http://gate.ac.uk/), and is free
7    *  software, licenced under the GNU Library General Public License,
8    *  Version 2, June 1991 (in the distribution as file licence.html,
9    *  and also available at http://gate.ac.uk/gate/licence.html).
10   *
11   *  Marin Dimitrov, 16/Oct/2001
12   *
13   *  $Id: DatabaseDocumentImpl.java,v 1.60 2002/07/12 13:24:28 valyt Exp $
14   */
15  
16  package gate.corpora;
17  
18  
19  import java.sql.*;
20  import java.io.*;
21  import java.util.*;
22  import java.net.*;
23  
24  import oracle.jdbc.driver.*;
25  import junit.framework.*;
26  
27  import gate.*;
28  import gate.util.*;
29  import gate.persist.*;
30  import gate.annotation.*;
31  import gate.creole.*;
32  import gate.event.*;
33  
34  public class DatabaseDocumentImpl extends DocumentImpl
35                                    implements  //DatastoreListener,
36                                                //Document,
37                                                EventAwareDocument {
38  
39    private static final boolean DEBUG = false;
40  
41    private boolean     isContentRead;
42    private Object      contentLock;
43    private Connection  jdbcConn;
44    private String      jdbcSchema;
45    protected int       dbType;
46  
47    private boolean     contentChanged;
48    private boolean     featuresChanged;
49    private boolean     nameChanged;
50    private boolean     documentChanged;
51  
52    private Collection  removedAnotationSets;
53    private Collection  addedAnotationSets;
54  
55    private Document    parentDocument;
56    private int         maxAnnotationId;
57  
58    /**
59     * The listener for the events coming from the features.
60     */
61    protected EventsHandler eventHandler;
62  
63  
64    public DatabaseDocumentImpl() {
65  
66      //super();
67      contentLock = new Object();
68  
69      this.namedAnnotSets = new HashMap();
70  //    this.defaultAnnots = new DatabaseAnnotationSetImpl(this);
71  
72      this.isContentRead = false;
73  
74      this.contentChanged = false;
75      this.featuresChanged = false;
76      this.nameChanged = false;
77      this.documentChanged = false;
78  
79      this.removedAnotationSets = new Vector();
80      this.addedAnotationSets = new Vector();
81  
82      parentDocument = null;
83    }
84  
85    private void setDatabaseInfo(Connection conn)
86      throws PersistenceException {
87  
88      String url = null;
89  
90      try {
91        url = conn.getMetaData().getURL();
92      }
93      catch(SQLException sqle) {
94        throw new PersistenceException("cannot get jdbc metadata: ["+sqle.getMessage()+"]");
95      }
96  
97      this.jdbcSchema = DBHelper.getSchemaPrefix(url);
98      this.dbType = DBHelper.getDatabaseType(url);
99      Assert.assertNotNull(this.jdbcSchema);
100     Assert.assertTrue(this.dbType == DBHelper.ORACLE_DB ||
101                       this.dbType == DBHelper.POSTGRES_DB);
102 
103   }
104 
105 
106   public DatabaseDocumentImpl(Connection conn)
107     throws PersistenceException {
108 
109     //super();
110     contentLock = new Object();
111 
112     this.namedAnnotSets = new HashMap();
113 //    this.defaultAnnots = new DatabaseAnnotationSetImpl(this);
114 
115     this.isContentRead = false;
116     this.jdbcConn = conn;
117     setDatabaseInfo(this.jdbcConn);
118 
119     this.contentChanged = false;
120     this.featuresChanged = false;
121     this.nameChanged = false;
122     this.documentChanged = false;
123 
124     this.removedAnotationSets = new Vector();
125     this.addedAnotationSets = new Vector();
126 
127     parentDocument = null;
128   }
129 
130 
131 /*  public DatabaseDocumentImpl(Connection _conn,
132                               String _name,
133                               DatabaseDataStore _ds,
134                               Long _persistenceID,
135                               DocumentContent _content,
136                               FeatureMap _features,
137                               Boolean _isMarkupAware,
138                               URL _sourceURL,
139                               Long _urlStartOffset,
140                               Long _urlEndOffset,
141                               AnnotationSet _default,
142                               Map _named) {
143 
144     //this.jdbcConn =  _conn;
145     this(_conn);
146 
147     this.name = _name;
148     this.dataStore = _ds;
149     this.lrPersistentId = _persistenceID;
150     this.content = _content;
151     this.isContentRead = true;
152     this.features = _features;
153     this.markupAware = _isMarkupAware;
154     this.sourceUrl = _sourceURL;
155     this.sourceUrlStartOffset = _urlStartOffset;
156     this.sourceUrlEndOffset = _urlEndOffset;
157 
158     //annotations
159     //1. default
160     _setAnnotations(null,_default);
161 
162     //2. named (if any)
163     if (null != _named) {
164       Iterator itNamed = _named.values().iterator();
165       while (itNamed.hasNext()){
166         AnnotationSet currSet = (AnnotationSet)itNamed.next();
167         //add them all to the DBAnnotationSet
168         _setAnnotations(currSet.getName(),currSet);
169       }
170     }
171 
172     //3. add the listeners for the features
173     if (eventHandler == null)
174       eventHandler = new EventsHandler();
175     this.features.addFeatureMapListener(eventHandler);
176 
177     //4. add self as listener for the data store, so that we'll know when the DS is
178     //synced and we'll clear the isXXXChanged flags
179     this.dataStore.addDatastoreListener(this);
180   }
181 */
182 
183   /** The content of the document: a String for text; MPEG for video; etc. */
184   public DocumentContent getContent() {
185 
186     //1. if this is a child document then return the content of the parent resource
187     if (null != this.parentDocument) {
188       return this.parentDocument.getContent();
189     }
190     else {
191       //2. assert that no one is reading from DB now
192       synchronized(this.contentLock) {
193         if (false == this.isContentRead) {
194           _readContent();
195           this.isContentRead = true;
196         }
197       }
198 
199       //return content
200       return super.getContent();
201     }
202   }
203 
204   private void _readContent() {
205 
206     //preconditions
207     if (null == getLRPersistenceId()) {
208       throw new GateRuntimeException("can't construct a DatabaseDocument - not associated " +
209                                     " with any data store");
210     }
211 
212     if (false == getLRPersistenceId() instanceof Long) {
213       throw new GateRuntimeException("can't construct a DatabaseDocument -  " +
214                                       " invalid persistence ID");
215     }
216 
217     Long lrID = (Long)getLRPersistenceId();
218     //0. preconditions
219     Assert.assertNotNull(lrID);
220     Assert.assertTrue(false == this.isContentRead);
221     Assert.assertNotNull(this.content);
222 
223     //1. read from DB
224     PreparedStatement pstmt = null;
225     ResultSet rs = null;
226 
227     try {
228 
229       String sql = " select v1.enc_name, " +
230                    "        v1.dc_character_content, " +
231                    "        v1.dc_binary_content, " +
232                    "        v1.dc_content_type " +
233                    " from  "+this.jdbcSchema+"v_content v1 " +
234                    " where  v1.lr_id = ? ";
235       pstmt = this.jdbcConn.prepareStatement(sql);
236       pstmt.setLong(1,lrID.longValue());
237       pstmt.execute();
238       rs = pstmt.getResultSet();
239 
240       if (false == rs.next()) {
241         throw new SynchronisationException("empty reault set");
242       }
243 
244       if (this.dbType == DBHelper.ORACLE_DB) {
245 
246         String encoding = rs.getString("enc_name");
247         if (encoding.equals(DBHelper.DUMMY_ENCODING)) {
248           //no encoding was specified for this document
249           encoding = "";
250         }
251         Clob   clb = rs.getClob("dc_character_content");
252         Blob   blb = rs.getBlob("dc_binary_content");
253         long   contentType = rs.getLong("dc_content_type");
254 
255         Assert.assertTrue(DBHelper.CHARACTER_CONTENT == contentType);
256 
257         StringBuffer buff = new StringBuffer();
258         OracleDataStore.readCLOB(clb,buff);
259 
260         //2. set data members that were not previously initialized
261         this.content = new DocumentContentImpl(buff.toString());
262         this.encoding = encoding;
263       }
264 
265       else if (this.dbType == DBHelper.POSTGRES_DB) {
266 
267         String encoding = rs.getString("enc_name");
268         if (encoding.equals(DBHelper.DUMMY_ENCODING)) {
269           //no encoding was specified for this document
270           encoding = "";
271         }
272 
273         String content = rs.getString("dc_character_content");
274         long   contentType = rs.getLong("dc_content_type");
275 
276         Assert.assertTrue(DBHelper.CHARACTER_CONTENT == contentType);
277 
278         //2. set data members that were not previously initialized
279         this.content = new DocumentContentImpl(content);
280         this.encoding = encoding;
281       }
282 
283       else {
284         Assert.fail();
285       }
286     }
287     catch(SQLException sqle) {
288       throw new SynchronisationException("can't read content from DB: ["+ sqle.getMessage()+"]");
289     }
290     catch(IOException ioe) {
291       throw new SynchronisationException(ioe);
292     }
293     finally {
294       try {
295         DBHelper.cleanup(rs);
296         DBHelper.cleanup(pstmt);
297       }
298       catch(PersistenceException pe) {
299         throw new SynchronisationException("JDBC error: ["+ pe.getMessage()+"]");
300       }
301     }
302   }
303 
304 
305   /** Get the encoding of the document content source */
306   public String getEncoding() {
307 
308     //1. assert that no one is reading from DB now
309     synchronized(this.contentLock) {
310       if (false == this.isContentRead) {
311         _readContent();
312 
313         this.isContentRead = true;
314       }
315     }
316 
317     return super.getEncoding();
318   }
319 
320   /** Returns a map with the named annotation sets. It returns <code>null</code>
321    *  if no named annotaton set exists. */
322   public Map getNamedAnnotationSets() {
323 
324     Vector annNames = new Vector();
325 
326     PreparedStatement pstmt = null;
327     ResultSet rs = null;
328 
329     //1. get the names of all sets
330     try {
331       String sql = " select as_name " +
332                    " from  "+this.jdbcSchema+"v_annotation_set " +
333                    " where  lr_id = ? " +
334                    "  and as_name is not null";
335 
336       pstmt = this.jdbcConn.prepareStatement(sql);
337       pstmt.setLong(1,((Long)this.lrPersistentId).longValue());
338       pstmt.execute();
339       rs = pstmt.getResultSet();
340 
341       while (rs.next()) {
342         annNames.add(rs.getString("as_name"));
343       }
344     }
345     catch(SQLException sqle) {
346       throw new SynchronisationException("can't get named annotatios: ["+ sqle.getMessage()+"]");
347     }
348     finally {
349       try {
350         DBHelper.cleanup(rs);
351         DBHelper.cleanup(pstmt);
352       }
353       catch(PersistenceException pe) {
354         throw new SynchronisationException("JDBC error: ["+ pe.getMessage()+"]");
355       }
356     }
357 
358     //2. read annotations
359     for (int i=0; i< annNames.size(); i++) {
360       //delegate because of the data is already read getAnnotations() will just return
361       getAnnotations((String)annNames.elementAt(i));
362     }
363 
364     //3. delegate to the parent method
365     return super.getNamedAnnotationSets();
366 
367   } // getNamedAnnotationSets
368 
369 
370   /** Get the default set of annotations. The set is created if it
371     * doesn't exist yet.
372     */
373   public AnnotationSet getAnnotations() {
374 
375     //1. read from DB
376     _getAnnotations(null);
377 
378     //2. is there such set in the DB?
379     if (null == this.defaultAnnots) {
380       //create a DatabaseAnnotationSetImpl
381       //NOTE: we create the set and then delegate to the super mehtod, otherwise
382       //the super mehtod will create AnnotationSetImpl instead of DatabaseAnnotationSetImpl
383       //which will not work with DatabaseDocumentImpl
384       AnnotationSet aset = new DatabaseAnnotationSetImpl(this);
385 
386       //set internal member
387       this.defaultAnnots = aset;
388 
389       //3. fire events
390       fireAnnotationSetAdded(new DocumentEvent(this,
391                                                 DocumentEvent.ANNOTATION_SET_ADDED,
392                                                 null));
393     }
394 
395     //4. delegate
396     return super.getAnnotations();
397   } // getAnnotations()
398 
399 
400   /** Get a named set of annotations. Creates a new set if one with this
401     * name doesn't exist yet.
402     * If the provided name is null then it returns the default annotation set.
403     */
404   public AnnotationSet getAnnotations(String name) {
405 
406     //0. preconditions
407     Assert.assertNotNull(name);
408 
409     //1. read from DB if the set is there at all
410     _getAnnotations(name);
411 
412     //2. is there such set in the DB?
413     if (false == this.namedAnnotSets.keySet().contains(name)) {
414       //create a DatabaseAnnotationSetImpl
415       //NOTE: we create the set and then delegate to the super mehtod, otherwise
416       //the super mehtod will create AnnotationSetImpl instead of DatabaseAnnotationSetImpl
417       //which will not work with DatabaseDocumentImpl
418       AnnotationSet aset = new DatabaseAnnotationSetImpl(this,name);
419 
420       //add to internal collection
421       this.namedAnnotSets.put(name,aset);
422 
423       //add the set name to the list with the recently created sets
424       this.addedAnotationSets.add(name);
425 
426       //3. fire events
427       DocumentEvent evt = new DocumentEvent(this, DocumentEvent.ANNOTATION_SET_ADDED, name);
428       fireAnnotationSetAdded(evt);
429     }
430 
431     //3. delegate
432     return super.getAnnotations(name);
433   }
434 
435 
436   private void _getAnnotations(String name) {
437 
438     AnnotationSet as = null;
439 
440     //preconditions
441     if (null == getLRPersistenceId()) {
442       throw new GateRuntimeException("can't construct a DatabaseDocument - not associated " +
443                                     " with any data store");
444     }
445 
446     if (false == getLRPersistenceId() instanceof Long) {
447       throw new GateRuntimeException("can't construct a DatabaseDocument -  " +
448                                       " invalid persistence ID");
449     }
450 
451     //have we already read this set?
452 
453     if (null == name) {
454       //default set
455       if (this.defaultAnnots != null) {
456         //the default set is alredy read - do nothing
457         //super methods will take care
458         return;
459       }
460     }
461     else {
462       //named set
463       if (this.namedAnnotSets.containsKey(name)) {
464         //we've already read it - do nothing
465         //super methods will take care
466         return;
467       }
468     }
469 
470     Long lrID = (Long)getLRPersistenceId();
471     Long asetID = null;
472     //0. preconditions
473     Assert.assertNotNull(lrID);
474 
475     //1. read a-set info
476     PreparedStatement pstmt = null;
477     ResultSet rs = null;
478     try {
479       String sql = " select as_id " +
480                    " from  "+this.jdbcSchema+"v_annotation_set " +
481                    " where  lr_id = ? ";
482       //do we have aset name?
483       String clause = null;
484       if (null != name) {
485         clause =   "        and as_name = ? ";
486       }
487       else {
488         clause =   "        and as_name is null ";
489       }
490       sql = sql + clause;
491 
492       pstmt = this.jdbcConn.prepareStatement(sql);
493       pstmt.setLong(1,lrID.longValue());
494       if (null != name) {
495         pstmt.setString(2,name);
496       }
497       pstmt.execute();
498       rs = pstmt.getResultSet();
499 
500       if (rs.next()) {
501         //ok, there is such aset in the DB
502         asetID = new Long(rs.getLong(1));
503       }
504       else {
505         //wow, there is no such aset, so create new ...
506         //... by delegating to the super method
507         return;
508       }
509 
510       //1.5 cleanup
511       DBHelper.cleanup(rs);
512       DBHelper.cleanup(pstmt);
513 
514       //2. read annotation Features
515       HashMap featuresByAnnotationID = _readFeatures(asetID);
516 
517       //3. read annotations
518       AnnotationSetImpl transSet = new AnnotationSetImpl(this);
519 
520       String hint;
521 
522       if (this.dbType == DBHelper.ORACLE_DB) {
523         hint = "/*+ use_nl(v.t_annotation v.t_as_annotation) " +
524               "     use_nl(v.t_annotation_type v.t_annotation) "+
525               " */";
526       }
527       else {
528         hint = "";
529       }
530 
531       String sql1 = " select "+hint+
532                     "        ann_local_id, " +
533                     "        at_name, " +
534                     "        start_offset, " +
535                     "        end_offset " +
536                     " from  "+this.jdbcSchema+"v_annotation  v" +
537                     " where  asann_as_id = ? ";
538 
539       if (DEBUG) Out.println(">>>>> asetID=["+asetID+"]");
540 
541       pstmt = this.jdbcConn.prepareStatement(sql1);
542       pstmt.setLong(1,asetID.longValue());
543 
544       if (this.dbType == DBHelper.ORACLE_DB) {
545         ((OraclePreparedStatement)pstmt).setRowPrefetch(DBHelper.CHINK_SIZE_LARGE);
546       }
547       pstmt.execute();
548       rs = pstmt.getResultSet();
549 
550       while (rs.next()) {
551         //1. read data memebers
552         Integer annID = new Integer(rs.getInt(1));
553         String type = rs.getString(2);
554         Long startOffset = new Long(rs.getLong(3));
555         Long endOffset = new Long(rs.getLong(4));
556 
557         if (DEBUG) Out.println("ann_local_id=["+annID+"]");
558         if (DEBUG) Out.println("start_off=["+startOffset+"]");
559         if (DEBUG) Out.println("end_off=["+endOffset+"]");
560 
561         //2. get the features
562         FeatureMap fm = (FeatureMap)featuresByAnnotationID.get(annID);
563         //fm should NOT be null
564         if (null == fm) {
565           fm =  new SimpleFeatureMapImpl();
566         }
567 
568         //3. add to annotation set
569         transSet.add(annID,startOffset,endOffset,type,fm);
570       }//while
571 
572       //1.5, create a-set
573       if (null == name) {
574         as = new DatabaseAnnotationSetImpl(this, transSet);
575       }
576       else {
577         as = new DatabaseAnnotationSetImpl(this,name, transSet);
578       }
579     }
580     catch(SQLException sqle) {
581       throw new SynchronisationException("can't read annotations from DB: ["+ sqle.getMessage()+"]");
582     }
583     catch(InvalidOffsetException oe) {
584       throw new SynchronisationException(oe);
585     }
586     catch(PersistenceException pe) {
587       throw new SynchronisationException("JDBC error: ["+ pe.getMessage()+"]");
588     }
589     finally {
590       try {
591         DBHelper.cleanup(rs);
592         DBHelper.cleanup(pstmt);
593       }
594       catch(PersistenceException pe) {
595         throw new SynchronisationException("JDBC error: ["+ pe.getMessage()+"]");
596       }
597     }
598 
599 
600     //4. update internal data members
601     if (name == null) {
602       //default as
603       this.defaultAnnots = as;
604     }
605     else {
606       //named as
607       this.namedAnnotSets.put(name,as);
608     }
609 
610     //don't return the new aset, the super method will take care
611     return;
612   }
613 
614 
615 
616 
617   private HashMap _readFeatures(Long asetID) {
618 
619     PreparedStatement pstmt = null;
620     ResultSet rs = null;
621 
622     //1
623     String      prevKey = DBHelper.DUMMY_FEATURE_KEY;
624     String      currKey = null;
625 
626     Integer     prevAnnID = null;
627     Integer     currAnnID = null;
628 
629     Object      currFeatureValue = null;
630     Vector      currFeatureArray = new Vector();
631 
632     HashMap     currFeatures = new HashMap();
633     FeatureMap  annFeatures = null;
634 
635     HashMap     featuresByAnnotID = new HashMap();
636 
637     //2. read the features from DB
638 
639     try {
640 
641       if (this.dbType == DBHelper.ORACLE_DB) {
642         String sql = " select /*+ use_nl(v.t_annotation v.t_as_annotation) "+
643                      "            use_nl(v.t_feature v.t_annotation) "+
644                      "            index(v.t_feature xt_feature_01) "+
645                      "            use_nl(v.t_feature_key v.t_feature) "+
646                      "           full(v.t_feature_key)           "+
647                      "        */                                  "+
648                      "                                            " +
649                      "        ann_local_id, " +
650                      "        key, " +
651                      "        ft_value_type, " +
652                      "        ft_number_value, " +
653                      "        ft_character_value, " +
654                      "        ft_long_character_value, " +
655                      "        ft_binary_value " +
656                      " from  "+this.jdbcSchema+"v_annotation_features v" +
657                      " where  set_id = ? " +
658                      " order by ann_local_id,key ";
659 
660         pstmt = this.jdbcConn.prepareStatement(sql);
661         pstmt.setLong(1,asetID.longValue());
662         ((OraclePreparedStatement)pstmt).setRowPrefetch(DBHelper.CHINK_SIZE_LARGE);
663         pstmt.execute();
664         rs = pstmt.getResultSet();
665       }
666 
667       else if (this.dbType == DBHelper.POSTGRES_DB) {
668 
669         String sql = " select " +
670                      "        ann_local_id, " +
671                      "        key, " +
672                      "        ft_value_type, " +
673                      "        ft_int_value, " +
674                      "        ft_float_value, " +
675                      "        ft_character_value, " +
676                      "        ft_binary_value " +
677                      " from  "+this.jdbcSchema+"v_annotation_features " +
678                      " where  set_id = ? " +
679                      " order by ann_local_id,key ";
680 
681         pstmt = this.jdbcConn.prepareStatement(sql);
682         pstmt.setLong(1,asetID.longValue());
683         pstmt.execute();
684         rs = pstmt.getResultSet();
685       }
686 
687       else {
688         Assert.fail();
689       }
690 
691       while (rs.next()) {
692         //NOTE: because there are LOBs in the resulset
693         //the columns should be read in the order they appear
694         //in the query
695 
696         prevAnnID = currAnnID;
697         currAnnID = new Integer(rs.getInt("ann_local_id"));
698 
699         //2.1 is this a new Annotation?
700         if (!currAnnID.equals(prevAnnID) && prevAnnID != null) {
701           //new one
702           //2.1.1 normalize the hashmap with the features, and add
703           //the elements into a new FeatureMap
704           annFeatures = new SimpleFeatureMapImpl();
705           Set entries = currFeatures.entrySet();
706           Iterator itFeatureArrays = entries.iterator();
707 
708           while(itFeatureArrays.hasNext()) {
709             Map.Entry currEntry = (Map.Entry)itFeatureArrays.next();
710             String key = (String)currEntry.getKey();
711             Vector val = (Vector)currEntry.getValue();
712 
713             //add to feature map normalized array
714             Assert.assertTrue(val.size() >= 1);
715 
716             if (val.size() == 1) {
717               //the single elemnt of the array
718               annFeatures.put(key,val.firstElement());
719             }
720             else {
721               //the whole array
722               annFeatures.put(key,val);
723             }
724           }//while
725 
726           //2.1.2. add the featuremap for this annotation to the hashmap
727           featuresByAnnotID.put(prevAnnID,annFeatures);
728           //2.1.3. clear temp hashtable with feature vectors
729           currFeatures.clear();
730 /*??*/          prevAnnID = currAnnID;
731         }//if -- is new annotation
732 
733         currKey = rs.getString("key");
734         Long valueType = new Long(rs.getLong("ft_value_type"));
735 
736         //we don't quite know what is the type of the NUMBER
737         //stored in DB
738         Object numberValue = null;
739 
740         //for all numeric types + boolean -> read from DB as appropriate
741         //Java object
742         switch(valueType.intValue()) {
743 
744           case DBHelper.VALUE_TYPE_BOOLEAN:
745 
746             if (this.dbType == DBHelper.ORACLE_DB) {
747               numberValue = new Boolean(rs.getBoolean("ft_number_value"));
748             }
749             else if (this.dbType == DBHelper.POSTGRES_DB){
750               numberValue = new Boolean(rs.getBoolean("ft_int_value"));
751             }
752             else {
753               Assert.fail();
754             }
755 
756             break;
757 
758 
759           case DBHelper.VALUE_TYPE_FLOAT:
760 
761             if (this.dbType == DBHelper.ORACLE_DB) {
762               numberValue = new Float(rs.getFloat("ft_number_value"));
763             }
764             else if (this.dbType == DBHelper.POSTGRES_DB){
765               numberValue = new Float(rs.getFloat("ft_float_value"));
766             }
767             else {
768               Assert.fail();
769             }
770 
771             break;
772 
773           case DBHelper.VALUE_TYPE_INTEGER:
774 
775             if (this.dbType == DBHelper.ORACLE_DB) {
776               numberValue = new Integer(rs.getInt("ft_number_value"));
777             }
778             else if (this.dbType == DBHelper.POSTGRES_DB){
779               numberValue = new Integer(rs.getInt("ft_int_value"));
780             }
781             else {
782               Assert.fail();
783             }
784 
785             break;
786 
787           case DBHelper.VALUE_TYPE_LONG:
788 
789             if (this.dbType == DBHelper.ORACLE_DB) {
790               numberValue = new Long(rs.getLong("ft_number_value"));
791             }
792             else if (this.dbType == DBHelper.POSTGRES_DB){
793               numberValue = new Long(rs.getLong("ft_int_value"));
794             }
795             else {
796               Assert.fail();
797             }
798 
799             break;
800 
801           default:
802             //do nothing, will be handled in the next switch statement
803         }
804 
805         //don't forget to read the rest of the current row
806         String stringValue = rs.getString("ft_character_value");
807         Clob clobValue = null;
808         Blob blobValue = null;
809 
810         if (this.dbType == DBHelper.ORACLE_DB) {
811           clobValue = rs.getClob("ft_long_character_value");
812           blobValue = rs.getBlob("ft_binary_value");
813         }
814 
815         switch(valueType.intValue()) {
816 
817           case DBHelper.VALUE_TYPE_NULL:
818             currFeatureValue = null;
819             break;
820 
821           case DBHelper.VALUE_TYPE_BINARY:
822             throw new MethodNotImplementedException();
823 
824           case DBHelper.VALUE_TYPE_BOOLEAN:
825           case DBHelper.VALUE_TYPE_FLOAT:
826           case DBHelper.VALUE_TYPE_INTEGER:
827           case DBHelper.VALUE_TYPE_LONG:
828             currFeatureValue = numberValue;
829             break;
830 
831           case DBHelper.VALUE_TYPE_STRING:
832 
833             if (this.dbType == DBHelper.ORACLE_DB && null == stringValue) {
834               //this one is tricky too
835               //if the string is < 4000 bytes long then it's stored as varchar2
836               //otherwise as CLOB
837 
838               StringBuffer temp = new StringBuffer();
839               OracleDataStore.readCLOB(clobValue,temp);
840               currFeatureValue = temp.toString();
841             }
842             else { /* PostgresDB or (Oracle DB + value is stored in varchar column) */
843               currFeatureValue = stringValue;
844             }
845             break;
846 
847           default:
848             throw new SynchronisationException("Invalid feature type found in DB, value is ["+valueType+"]");
849         }//switch
850 
851         //ok, we got the key/value pair now
852         //2.2 is this a new feature key?
853         if (false == currFeatures.containsKey(currKey)) {
854           //new key
855           Vector keyValue = new Vector();
856           keyValue.add(currFeatureValue);
857           currFeatures.put(currKey,keyValue);
858         }
859         else {
860           //key is present, append to existing vector
861           ((Vector)currFeatures.get(currKey)).add(currFeatureValue);
862         }
863 
864         prevKey = currKey;
865       }//while
866 
867 
868       //2.3 process the last Annotation left
869       annFeatures = new SimpleFeatureMapImpl();
870 
871       Set entries = currFeatures.entrySet();
872       Iterator itFeatureArrays = entries.iterator();
873 
874       while(itFeatureArrays.hasNext()) {
875         Map.Entry currEntry = (Map.Entry)itFeatureArrays.next();
876         String key = (String)currEntry.getKey();
877         Vector val = (Vector)currEntry.getValue();
878 
879         //add to feature map normalized array
880         Assert.assertTrue(val.size() >= 1);
881 
882         if (val.size() == 1) {
883           //the single elemnt of the array
884           annFeatures.put(key,val.firstElement());
885         }
886         else {
887           //the whole array
888           annFeatures.put(key,val);
889         }
890       }//while
891 
892       //2.3.1. add the featuremap for this annotation to the hashmap
893       if (null != currAnnID) {
894         // do we have features at all for this annotation?
895         featuresByAnnotID.put(currAnnID,annFeatures);
896       }
897 
898       //3. return the hashmap
899       return featuresByAnnotID;
900     }
901     catch(SQLException sqle) {
902       throw new SynchronisationException("can't read content from DB: ["+ sqle.getMessage()+"]");
903     }
904     catch(IOException sqle) {
905       throw new SynchronisationException("can't read content from DB: ["+ sqle.getMessage()+"]");
906     }
907     finally {
908       try {
909         DBHelper.cleanup(rs);
910         DBHelper.cleanup(pstmt);
911       }
912       catch(PersistenceException pe) {
913         throw new SynchronisationException("JDBC error: ["+ pe.getMessage()+"]");
914       }
915     }
916   }
917 
918 
919   /** Set method for the document content */
920   public void setContent(DocumentContent content) {
921 
922     //if the document is a child document then setContent()is prohibited
923     if (null != this.parentDocument) {
924       Err.prln("content of document ["+this.name+"] cannot be changed!");
925       return;
926     }
927     else {
928       super.setContent(content);
929       this.contentChanged = true;
930     }
931   }
932 
933   /** Set the feature set */
934   public void setFeatures(FeatureMap features) {
935     //1. save them first, so we can remove the listener
936     FeatureMap oldFeatures = this.features;
937 
938     super.setFeatures(features);
939 
940     this.featuresChanged = true;
941 
942     //4. sort out the listeners
943     if (eventHandler != null)
944       oldFeatures.removeFeatureMapListener(eventHandler);
945     else
946       eventHandler = new EventsHandler();
947     this.features.addFeatureMapListener(eventHandler);
948   }
949 
950   /** Sets the name of this resource*/
951   public void setName(String name){
952     super.setName(name);
953 
954     this.nameChanged = true;
955   }
956 
957 
958   private List getAnnotationsForOffset(AnnotationSet aDumpAnnotSet,Long offset){
959     throw new MethodNotImplementedException();
960   }
961 
962 
963   public void setNextNodeId(int nextID){
964     Assert.assertTrue(nextID >= 0);
965     this.nextNodeId = nextID;
966   }
967 
968 
969   public boolean isResourceChanged(int changeType) {
970 
971     switch(changeType) {
972 
973       case EventAwareLanguageResource.DOC_CONTENT:
974         return this.contentChanged;
975       case EventAwareLanguageResource.RES_FEATURES:
976         return this.featuresChanged;
977       case EventAwareLanguageResource.RES_NAME:
978         return this.nameChanged;
979       case EventAwareLanguageResource.DOC_MAIN:
980         return this.documentChanged;
981       default:
982         throw new IllegalArgumentException();
983     }
984 
985   }
986 
987   private void _setAnnotations(String setName,Collection annotations)
988     throws InvalidOffsetException {
989 
990     AnnotationSet tempSet = null;
991 
992     if (null == setName) {
993       Assert.assertTrue(null == this.defaultAnnots);
994 //      this.defaultAnnots = new DatabaseAnnotationSetImpl(this,annotations);
995       tempSet = new DatabaseAnnotationSetImpl(this);
996       this.defaultAnnots = tempSet;
997     }
998     else {
999       Assert.assertTrue(false == this.namedAnnotSets.containsKey(setName));
1000//      AnnotationSet annSet = new DatabaseAnnotationSetImpl(this,setName,annotations);
1001      tempSet = new DatabaseAnnotationSetImpl(this,setName);
1002      this.namedAnnotSets.put(setName,tempSet);
1003    }
1004
1005    //NOTE - the source aset is not from this document, so we can't use the proper constructor -
1006    //we should iterate all elements from the original aset and create equiva elements in the new aset
1007    Iterator itAnnotations = annotations.iterator();
1008    while (itAnnotations.hasNext()) {
1009      Annotation currAnn = (Annotation)itAnnotations.next();
1010      tempSet.add(currAnn.getId(),
1011                  currAnn.getStartNode().getOffset(),
1012                  currAnn.getEndNode().getOffset(),
1013                  currAnn.getType(),
1014                  currAnn.getFeatures());
1015
1016      //adjust the maxAnnotationID
1017      this.maxAnnotationId = (currAnn.getId().intValue() >= this.maxAnnotationId)
1018                              ? currAnn.getId().intValue()
1019                              : this.maxAnnotationId;
1020    }
1021
1022  }
1023
1024  /** Set method for the document's URL */
1025  public void setSourceUrl(URL sourceUrl) {
1026
1027    this.documentChanged = true;
1028    super.setSourceUrl(sourceUrl);
1029  } // setSourceUrl
1030
1031
1032  /** Documents may be packed within files; in this case an optional pair of
1033    * offsets refer to the location of the document. This method sets the
1034    * end offset.
1035    */
1036  public void setSourceUrlEndOffset(Long sourceUrlEndOffset) {
1037
1038    this.documentChanged = true;
1039    super.setSourceUrlEndOffset(sourceUrlEndOffset);
1040  } // setSourceUrlStartOffset
1041
1042
1043  /** Documents may be packed within files; in this case an optional pair of
1044    * offsets refer to the location of the document. This method sets the
1045    * start offset.
1046    */
1047  public void setSourceUrlStartOffset(Long sourceUrlStartOffset) {
1048
1049    this.documentChanged = true;
1050    super.setSourceUrlStartOffset(sourceUrlStartOffset);
1051  } // setSourceUrlStartOffset
1052
1053  /** Make the document markup-aware. This will trigger the creation
1054   *  of a DocumentFormat object at Document initialisation time; the
1055   *  DocumentFormat object will unpack the markup in the Document and
1056   *  add it as annotations. Documents are <B>not</B> markup-aware by default.
1057   *
1058   *  @param b markup awareness status.
1059   */
1060  public void setMarkupAware(Boolean newMarkupAware) {
1061
1062    this.documentChanged = true;
1063    super.setMarkupAware(newMarkupAware);
1064  }
1065
1066  /**
1067   * All the events from the features are handled by
1068   * this inner class.
1069   */
1070  class EventsHandler implements gate.event.FeatureMapListener {
1071    public void featureMapUpdated(){
1072      //tell the document that its features have been updated
1073      featuresChanged = true;
1074    }
1075  }
1076
1077  /**
1078   * Overriden to remove the features listener, when the document is closed.
1079   */
1080  public void cleanup() {
1081
1082    if (eventHandler != null)
1083
1084    this.features.removeFeatureMapListener(eventHandler);
1085    getDataStore().removeDatastoreListener(this);
1086
1087    //unregister annot-sets
1088    if (null != this.defaultAnnots) {
1089      getDataStore().removeDatastoreListener((DatastoreListener)this.defaultAnnots);
1090    }
1091
1092    Set loadedNamedAnnots = this.namedAnnotSets.entrySet();
1093    Iterator it = loadedNamedAnnots.iterator();
1094    while (it.hasNext()) {
1095      Map.Entry currEntry = (Map.Entry)it.next();
1096      AnnotationSet currSet = (AnnotationSet)currEntry.getValue();
1097      //unregister
1098      getDataStore().removeDatastoreListener((DatastoreListener)currSet);
1099    }
1100
1101    super.cleanup();
1102  }///inner class EventsHandler
1103
1104
1105  /**
1106   * Called by a datastore when a new resource has been adopted
1107   */
1108  public void resourceAdopted(DatastoreEvent evt){
1109  }
1110
1111  /**
1112   * Called by a datastore when a resource has been deleted
1113   */
1114  public void resourceDeleted(DatastoreEvent evt){
1115
1116    Assert.assertNotNull(evt);
1117    Assert.assertNotNull(evt.getResourceID());
1118
1119    //unregister self as listener from the DataStore
1120    if (evt.getResourceID().equals(this.getLRPersistenceId())) {
1121
1122      //someone deleted this document
1123      getDataStore().removeDatastoreListener(this);
1124
1125      //unregister annot-sets
1126      if (null != this.defaultAnnots) {
1127        getDataStore().removeDatastoreListener((DatastoreListener)this.defaultAnnots);
1128      }
1129
1130      Set loadedNamedAnnots = this.namedAnnotSets.entrySet();
1131      Iterator it = loadedNamedAnnots.iterator();
1132      while (it.hasNext()) {
1133        Map.Entry currEntry = (Map.Entry)it.next();
1134        AnnotationSet currSet = (AnnotationSet)currEntry.getValue();
1135        //unregister
1136        getDataStore().removeDatastoreListener((DatastoreListener)currSet);
1137      }
1138    }
1139  }//resourceDeleted
1140
1141  /**
1142   * Called by a datastore when a resource has been wrote into the datastore
1143   */
1144  public void resourceWritten(DatastoreEvent evt){
1145
1146    Assert.assertNotNull(evt);
1147    Assert.assertNotNull(evt.getResourceID());
1148
1149    //is the event for us?
1150    if (evt.getResourceID().equals(this.getLRPersistenceId())) {
1151      //wow, the event is for me
1152      //clear all flags, the content is synced with the DB
1153      this.contentChanged =
1154        this.documentChanged =
1155          this.featuresChanged =
1156            this.nameChanged = false;
1157
1158      this.removedAnotationSets.clear();
1159      this.addedAnotationSets.clear();
1160    }
1161
1162
1163  }
1164
1165  public Collection getLoadedAnnotationSets() {
1166
1167    //never return the data member - return a clone
1168    Assert.assertNotNull(this.namedAnnotSets);
1169    Vector result = new Vector(this.namedAnnotSets.values());
1170    if (null != this.defaultAnnots) {
1171      result.add(this.defaultAnnots);
1172    }
1173
1174    return result;
1175  }
1176
1177
1178  public Collection getRemovedAnnotationSets() {
1179
1180    //return a clone
1181    return new Vector(this.removedAnotationSets);
1182  }
1183
1184  public Collection getAddedAnnotationSets() {
1185
1186    //return a clone
1187    return new Vector(this.addedAnotationSets);
1188  }
1189
1190  public void removeAnnotationSet(String name) {
1191
1192    //1. add to the list of removed a-sets
1193    this.removedAnotationSets.add(name);
1194
1195    //if the set was read from the DB then it is registered as datastore listener and ...
1196    //there may be chnges in it
1197    //NOTE that default set cannot be reoved, so we just ignore it
1198
1199    if (this.namedAnnotSets.keySet().contains(name)) {
1200      //set was loaded
1201      AnnotationSet aset = (AnnotationSet)this.namedAnnotSets.get(name);
1202
1203      Assert.assertNotNull(aset);
1204      Assert.assertTrue(aset instanceof DatabaseAnnotationSetImpl);
1205
1206      //3. unregister it as a DataStoreListener
1207      this.dataStore.removeDatastoreListener((DatastoreListener)aset);
1208    }
1209
1210    //4. delegate
1211    super.removeAnnotationSet(name);
1212  }
1213
1214  /**
1215   * Returns true of an LR has been modified since the last sync.
1216   * Always returns false for transient LRs.
1217   */
1218  public boolean isModified() {
1219    return this.isResourceChanged(EventAwareLanguageResource.DOC_CONTENT) ||
1220            this.isResourceChanged(EventAwareLanguageResource.RES_FEATURES) ||
1221              this.isResourceChanged(EventAwareLanguageResource.RES_NAME) ||
1222                this.isResourceChanged(EventAwareLanguageResource.DOC_MAIN);
1223  }
1224
1225
1226  /**
1227   * Returns the parent LR of this LR.
1228   * Only relevant for LRs that support shadowing. Most do not by default.
1229   */
1230  public LanguageResource getParent()
1231    throws PersistenceException,SecurityException {
1232
1233    return this.parentDocument;
1234  }//getParent
1235
1236  /**
1237   * Sets the parent LR of this LR.
1238   * Only relevant for LRs that support shadowing. Most do not by default.
1239   */
1240  public void setParent(LanguageResource parentLR)
1241    throws PersistenceException,SecurityException {
1242
1243    //0. preconditions
1244    Assert.assertNotNull(parentLR);
1245
1246    if (false == parentLR instanceof DatabaseDocumentImpl) {
1247      throw new IllegalArgumentException("invalid parent resource set");
1248    }
1249
1250    //1.
1251    this.parentDocument = (Document)parentLR;
1252
1253  }//setParent
1254
1255  public void setInitData__$$__(Object data)
1256    throws PersistenceException, InvalidOffsetException {
1257
1258    HashMap initData = (HashMap)data;
1259
1260    this.jdbcConn = (Connection)initData.get("JDBC_CONN");
1261    setDatabaseInfo(this.jdbcConn);
1262    this.dataStore = (DatabaseDataStore)initData.get("DS");
1263    this.lrPersistentId = (Long)initData.get("LR_ID");
1264    this.name = (String)initData.get("DOC_NAME");
1265    this.content = (DocumentContent)initData.get("DOC_CONTENT");
1266    this.isContentRead = true;
1267    this.features = (FeatureMap)initData.get("DOC_FEATURES");
1268    this.markupAware = (Boolean)initData.get("DOC_MARKUP_AWARE");
1269    this.sourceUrl = (URL)initData.get("DOC_SOURCE_URL");
1270    this.sourceUrlStartOffset = (Long)initData.get("DOC_SOURCE_URL_START");
1271    this.sourceUrlEndOffset = (Long)initData.get("DOC_SOURCE_URL_END");
1272
1273    Integer nextNodeID = (Integer)initData.get("DOC_NEXT_NODE_ID");
1274    if (null != nextNodeID) {
1275      this.setNextNodeId(nextNodeID.intValue());
1276    }
1277
1278    Integer nextAnnID = (Integer)initData.get("DOC_NEXT_ANN_ID");
1279    if (null != nextAnnID) {
1280      this.setNextAnnotationId(nextAnnID.intValue());
1281    }
1282
1283    this.parentDocument = (Document)initData.get("PARENT_LR");
1284
1285    //annotations
1286    //1. default
1287    AnnotationSet _default = (AnnotationSet)initData.get("DOC_DEFAULT_ANNOTATIONS");
1288    if (null != _default) {
1289      _setAnnotations(null,_default);
1290    }
1291
1292    //2. named (if any)
1293    Map _named = (Map)initData.get("DOC_NAMED_ANNOTATION_SETS");
1294    if (null != _named) {
1295      Iterator itNamed = _named.values().iterator();
1296      while (itNamed.hasNext()){
1297        AnnotationSet currSet = (AnnotationSet)itNamed.next();
1298        //add them all to the DBAnnotationSet, except the ORIGINAL MARKUPS - handled in the super init()
1299        if (false == currSet.getName().equals(GateConstants.ORIGINAL_MARKUPS_ANNOT_SET_NAME)) {
1300          _setAnnotations(currSet.getName(),currSet);
1301        }
1302      }
1303    }
1304
1305    //3. add the listeners for the features (if any)
1306    if (null != this.features) {
1307      if (eventHandler == null)
1308        eventHandler = new EventsHandler();
1309      this.features.addFeatureMapListener(eventHandler);
1310    }
1311
1312    //4. add self as listener for the data store, so that we'll know when the DS is
1313    //synced and we'll clear the isXXXChanged flags
1314    if (null != this.dataStore) {
1315      this.dataStore.addDatastoreListener(this);
1316    }
1317
1318  }
1319
1320  public Object getInitData__$$__(Object initData) {
1321    return null;
1322  }
1323
1324  /** Initialise this resource, and return it. */
1325  public Resource init() throws ResourceInstantiationException {
1326
1327    Resource result = super.init();
1328
1329    if (this.nextAnnotationId <= this.maxAnnotationId) {
1330      this.nextAnnotationId = this.maxAnnotationId +1;
1331    }
1332
1333    return result;
1334  }
1335
1336}