CorpusQualityAssurance.java
0001 /*
0002  *  Copyright (c) 2009-2010, Ontotext AD.
0003  *  Copyright (c) 1995-2012, The University of Sheffield. See the file
0004  *  COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
0005  *
0006  *  This file is part of GATE (see http://gate.ac.uk/), and is free
0007  *  software, licenced under the GNU Library General Public License,
0008  *  Version 2, June 1991 (in the distribution as file licence.html,
0009  *  and also available at http://gate.ac.uk/gate/licence.html).
0010  *
0011  *  Thomas Heitz - 10 June 2009
0012  *
0013  *  $Id: CorpusQualityAssurance.java 17879 2014-04-18 16:59:35Z markagreenwood $
0014  */
0015 
0016 package gate.gui;
0017 
0018 import java.awt.BorderLayout;
0019 import java.awt.Cursor;
0020 import java.awt.Dimension;
0021 import java.awt.GridBagConstraints;
0022 import java.awt.GridBagLayout;
0023 import java.awt.Insets;
0024 import java.awt.event.ActionEvent;
0025 import java.awt.event.KeyEvent;
0026 import java.awt.event.MouseAdapter;
0027 import java.awt.event.MouseEvent;
0028 import java.io.BufferedWriter;
0029 import java.io.File;
0030 import java.io.FileWriter;
0031 import java.io.IOException;
0032 import java.io.Writer;
0033 import java.math.RoundingMode;
0034 import java.net.MalformedURLException;
0035 import java.net.URL;
0036 import java.util.*;
0037 import java.util.Timer;
0038 import java.text.NumberFormat;
0039 import java.text.Collator;
0040 
0041 import javax.swing.*;
0042 import javax.swing.event.AncestorEvent;
0043 import javax.swing.event.AncestorListener;
0044 import javax.swing.event.ChangeEvent;
0045 import javax.swing.event.ChangeListener;
0046 import javax.swing.event.ListSelectionEvent;
0047 import javax.swing.event.ListSelectionListener;
0048 import javax.swing.table.DefaultTableModel;
0049 import javax.swing.table.JTableHeader;
0050 import javax.swing.text.Position;
0051 
0052 import gate.Annotation;
0053 import gate.AnnotationSet;
0054 import gate.Corpus;
0055 import gate.Document;
0056 import gate.Factory;
0057 import gate.Gate;
0058 import gate.Resource;
0059 import gate.creole.AbstractVisualResource;
0060 import gate.event.CorpusEvent;
0061 import gate.creole.metadata.CreoleResource;
0062 import gate.creole.metadata.GuiType;
0063 import gate.event.CorpusListener;
0064 import gate.swing.XJTable;
0065 import gate.swing.XJFileChooser;
0066 import gate.util.AnnotationDiffer;
0067 import gate.util.ClassificationMeasures;
0068 import gate.util.OntologyMeasures;
0069 import gate.util.ExtensionFileFilter;
0070 import gate.util.OptionsMap;
0071 import gate.util.Strings;
0072 
0073 /**
0074  * Quality assurance corpus view.
0075  * Compare two sets of annotations with optionally their features
0076  * globally for each annotation and for each document inside a corpus
0077  * with different measures notably precision, recall and F1-score.
0078  */
0079 @SuppressWarnings({"serial","unchecked","rawtypes","deprecation"})
0080 @CreoleResource(name = "Corpus Quality Assurance", guiType = GuiType.LARGE,
0081     resourceDisplayed = "gate.Corpus", mainViewer = false,
0082     helpURL = "http://gate.ac.uk/userguide/sec:eval:corpusqualityassurance")
0083 public class CorpusQualityAssurance extends AbstractVisualResource
0084   implements CorpusListener {
0085 
0086   @Override
0087   public Resource init(){
0088     initLocalData();
0089     initGuiComponents();
0090     initListeners();
0091     return this;
0092   }
0093 
0094   protected void initLocalData(){
0095     collator = Collator.getInstance(Locale.ENGLISH);
0096     collator.setStrength(Collator.TERTIARY);
0097     documentTableModel = new DefaultTableModel();
0098     documentTableModel.addColumn("Document");
0099     documentTableModel.addColumn("Match");
0100     documentTableModel.addColumn("Only A");
0101     documentTableModel.addColumn("Only B");
0102     documentTableModel.addColumn("Overlap");
0103     annotationTableModel = new DefaultTableModel();
0104     annotationTableModel.addColumn("Annotation");
0105     annotationTableModel.addColumn("Match");
0106     annotationTableModel.addColumn("Only A");
0107     annotationTableModel.addColumn("Only B");
0108     annotationTableModel.addColumn("Overlap");
0109     document2TableModel = new DefaultTableModel();
0110     document2TableModel.addColumn("Document");
0111     document2TableModel.addColumn("Agreed");
0112     document2TableModel.addColumn("Total");
0113     confusionTableModel = new DefaultTableModel();
0114     types = new TreeSet<String>(collator);
0115     corpusChanged = false;
0116     measuresType = FSCORE_MEASURES;
0117     doubleComparator = new Comparator<String>() {
0118       @Override
0119       public int compare(String s1, String s2) {
0120         if (s1 == null || s2 == null) {
0121           return 0;
0122         else if (s1.equals("")) {
0123           return 1;
0124         else if (s2.equals("")) {
0125           return -1;
0126         else {
0127           return Double.valueOf(s1).compareTo(Double.valueOf(s2));
0128         }
0129       }
0130     };
0131     totalComparator = new Comparator<String>() {
0132       @Override
0133       public int compare(String s1, String s2) {
0134         if (s1 == null || s2 == null) {
0135           return 0;
0136         else if (s1.equals("Micro summary")) {
0137           return s2.equals("Macro summary"? -1;
0138         else if (s1.equals("Macro summary")) {
0139           return s2.equals("Micro summary"? -1;
0140         else if (s2.equals("Micro summary")) {
0141           return s1.equals("Macro summary": -1;
0142         else if (s2.equals("Macro summary")) {
0143           return s1.equals("Micro summary": -1;
0144         else {
0145           return s1.compareTo(s2);
0146         }
0147       }
0148     };
0149   }
0150 
0151   protected void initGuiComponents() {
0152     setLayout(new BorderLayout());
0153 
0154     JPanel sidePanel = new JPanel(new GridBagLayout());
0155     GridBagConstraints gbc = new GridBagConstraints();
0156     gbc.gridx = 0;
0157     sidePanel.add(Box.createVerticalStrut(5), gbc);
0158 
0159     // toolbar
0160     JToolBar toolbar = new JToolBar();
0161     toolbar.setFloatable(false);
0162     toolbar.add(openDocumentAction = new OpenDocumentAction());
0163     openDocumentAction.setEnabled(false);
0164     toolbar.add(openAnnotationDiffAction = new OpenAnnotationDiffAction());
0165     openAnnotationDiffAction.setEnabled(false);
0166     toolbar.add(exportToHtmlAction = new ExportToHtmlAction());
0167     toolbar.add(reloadCacheAction = new ReloadCacheAction());
0168     toolbar.add(new HelpAction());
0169     gbc.anchor = GridBagConstraints.NORTHWEST;
0170     sidePanel.add(toolbar, gbc);
0171     gbc.anchor = GridBagConstraints.NORTH;
0172     sidePanel.add(Box.createVerticalStrut(5), gbc);
0173 
0174     // annotation sets list
0175     JLabel label = new JLabel("Annotation Sets A/Key & B/Response");
0176     label.setToolTipText("aka 'Key & Response sets'");
0177     gbc.fill = GridBagConstraints.BOTH;
0178     sidePanel.add(label, gbc);
0179     sidePanel.add(Box.createVerticalStrut(2), gbc);
0180     setList = new JList();
0181     setList.setSelectionModel(new ToggleSelectionABModel(setList));
0182     setList.setPrototypeCellValue("present in every document");
0183     setList.setVisibleRowCount(4);
0184     gbc.weighty = 1;
0185     sidePanel.add(new JScrollPane(setList), gbc);
0186     gbc.weighty = 0;
0187     sidePanel.add(Box.createVerticalStrut(2), gbc);
0188     setCheck = new JCheckBox("present in every document"false);
0189     setCheck.addActionListener(new AbstractAction(){
0190       @Override
0191       public void actionPerformed(ActionEvent e) {
0192         updateSetList();
0193       }
0194     });
0195     sidePanel.add(setCheck, gbc);
0196     sidePanel.add(Box.createVerticalStrut(5), gbc);
0197 
0198     // annotation types list
0199     label = new JLabel("Annotation Types");
0200     label.setToolTipText("Annotation types to compare");
0201     sidePanel.add(label, gbc);
0202     sidePanel.add(Box.createVerticalStrut(2), gbc);
0203     typeList = new JList();
0204     typeList.setSelectionModel(new ToggleSelectionModel());
0205     typeList.setPrototypeCellValue("present in every document");
0206     typeList.setVisibleRowCount(4);
0207     gbc.weighty = 1;
0208     sidePanel.add(new JScrollPane(typeList), gbc);
0209     gbc.weighty = 0;
0210     sidePanel.add(Box.createVerticalStrut(2), gbc);
0211     typeCheck = new JCheckBox("present in every selected set"false);
0212     typeCheck.addActionListener(new AbstractAction(){
0213       @Override
0214       public void actionPerformed(ActionEvent e) {
0215         setList.getListSelectionListeners()[0].valueChanged(null);
0216       }
0217     });
0218     sidePanel.add(typeCheck, gbc);
0219     sidePanel.add(Box.createVerticalStrut(5), gbc);
0220 
0221     // annotation features list
0222     label = new JLabel("Annotation Features");
0223     label.setToolTipText("Annotation features to compare");
0224     sidePanel.add(label, gbc);
0225     sidePanel.add(Box.createVerticalStrut(2), gbc);
0226     featureList = new JList();
0227     featureList.setSelectionModel(new ToggleSelectionModel());
0228     featureList.setPrototypeCellValue("present in every document");
0229     featureList.setVisibleRowCount(4);
0230     gbc.weighty = 1;
0231     sidePanel.add(new JScrollPane(featureList), gbc);
0232     gbc.weighty = 0;
0233     sidePanel.add(Box.createVerticalStrut(2), gbc);
0234     featureCheck = new JCheckBox("present in every selected type"false);
0235     featureCheck.addActionListener(new AbstractAction(){
0236       @Override
0237       public void actionPerformed(ActionEvent e) {
0238         typeList.getListSelectionListeners()[0].valueChanged(null);
0239       }
0240     });
0241     sidePanel.add(featureCheck, gbc);
0242     sidePanel.add(Box.createVerticalStrut(5), gbc);
0243 
0244     // measures tabbed panes
0245     label = new JLabel("Measures");
0246     label.setToolTipText("Measures used to compare annotations");
0247     optionsButton = new JToggleButton("Options");
0248     optionsButton.setMargin(new Insets(1111));
0249     JPanel labelButtonPanel = new JPanel(new BorderLayout());
0250     labelButtonPanel.add(label, BorderLayout.WEST);
0251     labelButtonPanel.add(optionsButton, BorderLayout.EAST);
0252     sidePanel.add(labelButtonPanel, gbc);
0253     sidePanel.add(Box.createVerticalStrut(2), gbc);
0254     final JScrollPane measureScrollPane = new JScrollPane();
0255     measureList = new JList();
0256     measureList.setSelectionModel(new ToggleSelectionModel());
0257     String prefix = getClass().getName() '.';
0258     double beta = (userConfig.getDouble(prefix+"fscorebeta"== null?
0259       1.0 : userConfig.getDouble(prefix+"fscorebeta");
0260     double beta2 = (userConfig.getDouble(prefix+"fscorebeta2"== null?
0261       0.5 : userConfig.getDouble(prefix+"fscorebeta2");
0262     String fscore = "F" + beta + "-score ";
0263     String fscore2 = "F" + beta2 + "-score ";
0264     measureList.setModel(new ExtendedListModel(new String[]{
0265       fscore+"strict", fscore+"lenient", fscore+"average",
0266       fscore+"strict BDM", fscore+"lenient BDM", fscore+"average BDM",
0267       fscore2+"strict", fscore2+"lenient", fscore2+"average",
0268       fscore2+"strict BDM", fscore2+"lenient BDM", fscore2+"average BDM"}));
0269     measureList.setPrototypeCellValue("present in every document");
0270     measureList.setVisibleRowCount(4);
0271     measureScrollPane.setViewportView(measureList);
0272     final JScrollPane measure2ScrollPane = new JScrollPane();
0273     measure2List = new JList();
0274     measure2List.setSelectionModel(new ToggleSelectionModel());
0275     measure2List.setModel(new ExtendedListModel(new String[]{
0276       "Observed agreement""Cohen's Kappa" "Pi's Kappa"}));
0277     measure2List.setPrototypeCellValue("present in every document");
0278     measure2List.setVisibleRowCount(4);
0279     measure2ScrollPane.setViewportView(measure2List);
0280     measureTabbedPane = new JTabbedPane();
0281     measureTabbedPane.addTab("F-Score", null,
0282       measureScrollPane, "Inter-annotator agreement");
0283     measureTabbedPane.addTab("Classification", null,
0284       measure2ScrollPane, "Classification agreement");
0285     gbc.weighty = 1;
0286     sidePanel.add(measureTabbedPane, gbc);
0287     gbc.weighty = 0;
0288     sidePanel.add(Box.createVerticalStrut(5), gbc);
0289     sidePanel.add(Box.createVerticalGlue(), gbc);
0290 
0291     // options panel for fscore measures
0292     final JPanel measureOptionsPanel = new JPanel();
0293     measureOptionsPanel.setLayout(
0294       new BoxLayout(measureOptionsPanel, BoxLayout.Y_AXIS));
0295     JPanel betaPanel = new JPanel();
0296     betaPanel.setLayout(new BoxLayout(betaPanel, BoxLayout.X_AXIS));
0297     JLabel betaLabel = new JLabel("Fscore Beta 1:");
0298     final JSpinner betaSpinner =
0299       new JSpinner(new SpinnerNumberModel(beta, 010.1));
0300     betaSpinner.setToolTipText(
0301       "<html>Relative weight of precision and recall." +
0302       "<ul><li>1 weights equally precision and recall" +
0303       "<li>0.5 weights precision twice as much as recall" +
0304       "<li>2 weights recall twice as much as precision</ul></html>");
0305     betaPanel.add(betaLabel);
0306     betaPanel.add(Box.createHorizontalStrut(5));
0307     betaPanel.add(betaSpinner);
0308     betaPanel.add(Box.createHorizontalGlue());
0309     measureOptionsPanel.add(betaPanel);
0310     betaSpinner.setMaximumSize(new Dimension(Integer.MAX_VALUE,
0311       Math.round(betaLabel.getPreferredSize().height*1.5f)));
0312     JPanel beta2Panel = new JPanel();
0313     beta2Panel.setLayout(new BoxLayout(beta2Panel, BoxLayout.X_AXIS));
0314     JLabel beta2Label = new JLabel("Fscore Beta 2:");
0315     final JSpinner beta2Spinner =
0316       new JSpinner(new SpinnerNumberModel(beta2, 010.1));
0317     beta2Spinner.setToolTipText(betaSpinner.getToolTipText());
0318     beta2Panel.add(beta2Label);
0319     beta2Panel.add(Box.createHorizontalStrut(5));
0320     beta2Panel.add(beta2Spinner);
0321     beta2Panel.add(Box.createHorizontalGlue());
0322     measureOptionsPanel.add(beta2Panel);
0323     beta2Spinner.setMaximumSize(new Dimension(Integer.MAX_VALUE,
0324       Math.round(beta2Label.getPreferredSize().height*1.5f)));
0325     JPanel bdmFilePanel = new JPanel();
0326     bdmFilePanel.setLayout(new BoxLayout(bdmFilePanel, BoxLayout.X_AXIS));
0327     JLabel bdmFileLabel = new JLabel("BDM file:");
0328     JButton bdmFileButton = new JButton(new SetBdmFileAction());
0329     bdmFilePanel.add(bdmFileLabel);
0330     bdmFilePanel.add(Box.createHorizontalStrut(5));
0331     bdmFilePanel.add(bdmFileButton);
0332     bdmFilePanel.add(Box.createHorizontalGlue());
0333     measureOptionsPanel.add(bdmFilePanel);
0334 
0335     // options panel for classification measures
0336     final JPanel measure2OptionsPanel = new JPanel();
0337     measure2OptionsPanel.setLayout(
0338       new BoxLayout(measure2OptionsPanel, BoxLayout.Y_AXIS));
0339     JPanel verbosePanel = new JPanel();
0340     verbosePanel.setLayout(new BoxLayout(verbosePanel, BoxLayout.X_AXIS));
0341     boolean verbose = (userConfig.getBoolean(prefix+"verbose"== null?
0342       false : userConfig.getBoolean(prefix+"verbose");
0343     verboseOptionCheckBox = new JCheckBox("Output ignored annotations",verbose);
0344     verbosePanel.add(verboseOptionCheckBox);
0345     verbosePanel.add(Box.createHorizontalGlue());
0346     measure2OptionsPanel.add(verbosePanel);
0347 
0348     // options button action
0349     optionsButton.setAction(new AbstractAction("Options") {
0350       int[] selectedIndices;
0351       @Override
0352       public void actionPerformed(ActionEvent e) {
0353         JToggleButton button = (JToggleButtone.getSource();
0354         // switch measure options panel and measure list
0355         if (button.isSelected()) {
0356           if (measuresType == FSCORE_MEASURES) {
0357             selectedIndices = measureList.getSelectedIndices();
0358             measureScrollPane.setViewportView(measureOptionsPanel);
0359           else if (measuresType == CLASSIFICATION_MEASURES) {
0360             selectedIndices = measure2List.getSelectedIndices();
0361             measure2ScrollPane.setViewportView(measure2OptionsPanel);
0362           }
0363         else {
0364           String prefix = getClass().getEnclosingClass().getName() '.';
0365           if (measuresType == FSCORE_MEASURES) {
0366             // update beta with new values
0367             String fscore = "F" + betaSpinner.getValue() "-score ";
0368             String fscore2 = "F" + beta2Spinner.getValue() "-score ";
0369             measureList.setModel(new ExtendedListModel(new String[]{
0370               fscore+"strict", fscore+"lenient", fscore+"average",
0371               fscore+"strict BDM", fscore+"lenient BDM", fscore+"average BDM",
0372               fscore2+"strict", fscore2+"lenient", fscore2+"average",
0373               fscore2+"strict BDM", fscore2+"lenient BDM", fscore2+"average BDM"}));
0374             // save in GATE preferences
0375             userConfig.put(prefix+"fscorebeta", betaSpinner.getValue());
0376             userConfig.put(prefix+"fscorebeta2", beta2Spinner.getValue());
0377             // put back the list and its selection
0378             measureScrollPane.setViewportView(measureList);
0379             measureList.setSelectedIndices(selectedIndices);
0380           else if (measuresType == CLASSIFICATION_MEASURES) {
0381             userConfig.put(prefix+"verbose",verboseOptionCheckBox.isSelected());
0382             measure2ScrollPane.setViewportView(measure2List);
0383             measure2List.setSelectedIndices(selectedIndices);
0384           }
0385         }
0386       }
0387     });
0388 
0389     // compare button and progress bar
0390     JButton compareButton = new JButton(compareAction = new CompareAction());
0391     compareAction.setEnabled(false);
0392     sidePanel.add(compareButton, gbc);
0393     sidePanel.add(Box.createVerticalStrut(5), gbc);
0394     progressBar = new JProgressBar();
0395     progressBar.setStringPainted(true);
0396     progressBar.setString("");
0397     sidePanel.add(progressBar, gbc);
0398     sidePanel.add(Box.createVerticalStrut(5), gbc);
0399 
0400     // tables
0401     annotationTable = new XJTable() {
0402       @Override
0403       public boolean isCellEditable(int rowIndex, int vColIndex) {
0404         return false;
0405       }
0406       @Override
0407       protected JTableHeader createDefaultTableHeader() {
0408         return new JTableHeader(columnModel) {
0409           @Override
0410           public String getToolTipText(MouseEvent event) {
0411             int index = columnModel.getColumnIndexAtX(event.getPoint().x);
0412             if (index == -1) { return null}
0413             int modelIndex = columnModel.getColumn(index).getModelIndex();
0414             String columnName = this.table.getModel().getColumnName(modelIndex);
0415             return createToolTipFromColumnName(columnName);
0416           }
0417         };
0418       }
0419     };
0420     annotationTable.setModel(annotationTableModel);
0421     annotationTable.setSortable(false);
0422     annotationTable.setEnableHidingColumns(true);
0423     annotationTable.setAutoResizeMode(XJTable.AUTO_RESIZE_ALL_COLUMNS);
0424     documentTable = new XJTable() {
0425       @Override
0426       public boolean isCellEditable(int rowIndex, int vColIndex) {
0427         return false;
0428       }
0429       @Override
0430       protected JTableHeader createDefaultTableHeader() {
0431         return new JTableHeader(columnModel) {
0432           @Override
0433           public String getToolTipText(MouseEvent event) {
0434             int index = columnModel.getColumnIndexAtX(event.getPoint().x);
0435             if (index == -1) { return null}
0436             int modelIndex = columnModel.getColumn(index).getModelIndex();
0437             String columnName = this.table.getModel().getColumnName(modelIndex);
0438             return createToolTipFromColumnName(columnName);
0439           }
0440         };
0441       }
0442     };
0443     documentTable.setModel(documentTableModel);
0444     documentTable.setSortable(false);
0445     documentTable.setEnableHidingColumns(true);
0446     documentTable.setAutoResizeMode(XJTable.AUTO_RESIZE_ALL_COLUMNS);
0447     document2Table = new XJTable() {
0448       @Override
0449       public boolean isCellEditable(int rowIndex, int vColIndex) {
0450         return false;
0451       }
0452     };
0453     document2Table.setModel(document2TableModel);
0454     confusionTable = new XJTable() {
0455       @Override
0456       public boolean isCellEditable(int rowIndex, int vColIndex) {
0457         return false;
0458       }
0459     };
0460     confusionTable.setModel(confusionTableModel);
0461     confusionTable.setSortable(false);
0462 
0463     tableTabbedPane = new JTabbedPane();
0464     tableTabbedPane.addTab("Corpus statistics", null,
0465       new JScrollPane(annotationTable),
0466       "Compare each annotation type for the whole corpus");
0467     tableTabbedPane.addTab("Document statistics", null,
0468       new JScrollPane(documentTable),
0469       "Compare each documents in the corpus with theirs annotations");
0470 
0471     JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
0472     splitPane.setContinuousLayout(true);
0473     splitPane.setOneTouchExpandable(true);
0474     splitPane.setResizeWeight(0.80);
0475     splitPane.setLeftComponent(tableTabbedPane);
0476     splitPane.setRightComponent(new JScrollPane(sidePanel));
0477 
0478     add(splitPane);
0479   }
0480 
0481   protected void initListeners() {
0482 
0483     // when the view is shown update the tables if the corpus has changed
0484     addAncestorListener(new AncestorListener() {
0485       @Override
0486       public void ancestorAdded(AncestorEvent event) {
0487         if (!isShowing() || !corpusChanged) { return}
0488         if (timerTask != null) { timerTask.cancel()}
0489         Date timeToRun = new Date(System.currentTimeMillis() 1000);
0490         timerTask = new TimerTask() { @Override
0491         public void run() {
0492           readSetsTypesFeatures(0);
0493         }};
0494         timer.schedule(timerTask, timeToRun)// add a delay before updating
0495       }
0496       @Override
0497       public void ancestorRemoved(AncestorEvent event) { /* do nothing */ }
0498       @Override
0499       public void ancestorMoved(AncestorEvent event) { /* do nothing */ }
0500     });
0501 
0502     // when set list selection change
0503     setList.addListSelectionListener(new ListSelectionListener() {
0504       @Override
0505       public void valueChanged(ListSelectionEvent e) {
0506         if (typesSelected == null) {
0507           typesSelected = typeList.getSelectedValues();
0508         }
0509         typeList.setModel(new ExtendedListModel());
0510         keySetName = ((ToggleSelectionABModel)
0511           setList.getSelectionModel()).getSelectedValueA();
0512         responseSetName = ((ToggleSelectionABModel)
0513           setList.getSelectionModel()).getSelectedValueB();
0514         if (keySetName == null
0515          || responseSetName == null
0516          || setList.getSelectionModel().getValueIsAdjusting()) {
0517           compareAction.setEnabled(false);
0518           return;
0519         }
0520         setList.setEnabled(false);
0521         setCheck.setEnabled(false);
0522         // update type UI list
0523         TreeSet<String> someTypes = new TreeSet<String>();
0524         TreeMap<String, TreeSet<String>> typesFeatures;
0525         boolean firstLoop = true// needed for retainAll to work
0526         synchronized(docsSetsTypesFeatures) {
0527           for (TreeMap<String, TreeMap<String, TreeSet<String>>>
0528               setsTypesFeatures : docsSetsTypesFeatures.values()) {
0529             typesFeatures = setsTypesFeatures.get(
0530               keySetName.equals("[Default set]""" : keySetName);
0531             if (typesFeatures != null) {
0532               if (typeCheck.isSelected() && !firstLoop) {
0533                 someTypes.retainAll(typesFeatures.keySet());
0534               else {
0535                 someTypes.addAll(typesFeatures.keySet());
0536               }
0537             else if (typeCheck.isSelected()) {
0538               // empty set no types to display
0539               break;
0540             }
0541             typesFeatures = setsTypesFeatures.get(
0542               responseSetName.equals("[Default set]""" : responseSetName);
0543             if (typesFeatures != null) {
0544               if (typeCheck.isSelected()) {
0545                 someTypes.retainAll(typesFeatures.keySet());
0546               else {
0547                 someTypes.addAll(typesFeatures.keySet());
0548               }
0549             else if (typeCheck.isSelected()) {
0550               break;
0551             }
0552             firstLoop = false;
0553           }
0554         }
0555         typeList.setModel(new ExtendedListModel(someTypes.toArray()));
0556         if (someTypes.size() 0) {
0557           for (Object value : typesSelected) {
0558             // put back the selection if possible
0559             int index = typeList.getNextMatch(
0560               (Stringvalue, 0, Position.Bias.Forward);
0561             if (index != -1) {
0562               typeList.setSelectedIndex(index);
0563             }
0564           }
0565         }
0566         typesSelected = null;
0567         setList.setEnabled(true);
0568         setCheck.setEnabled(true);
0569         if (measuresType == FSCORE_MEASURES) {
0570           compareAction.setEnabled(true);
0571         }
0572       }
0573     });
0574 
0575     // when type list selection change
0576     typeList.addListSelectionListener(new ListSelectionListener() {
0577       @Override
0578       public void valueChanged(ListSelectionEvent e) {
0579         // update feature UI list
0580         if (featuresSelected == null) {
0581           featuresSelected = featureList.getSelectedValues();
0582         }
0583         featureList.setModel(new ExtendedListModel());
0584         if (typeList.getSelectedValues().length == 0
0585          || typeList.getSelectionModel().getValueIsAdjusting()) {
0586           return;
0587         }
0588         final Set<String> typeNames = new HashSet<String>();
0589         for (Object type : typeList.getSelectedValues()) {
0590           typeNames.add((Stringtype);
0591         }
0592         typeList.setEnabled(false);
0593         typeCheck.setEnabled(false);
0594         TreeSet<String> features = new TreeSet<String>(collator);
0595         TreeMap<String, TreeSet<String>> typesFeatures;
0596         boolean firstLoop = true// needed for retainAll to work
0597         synchronized(docsSetsTypesFeatures) {
0598           for (TreeMap<String, TreeMap<String, TreeSet<String>>> sets :
0599                docsSetsTypesFeatures.values()) {
0600             typesFeatures = sets.get(keySetName.equals("[Default set]"?
0601               "" : keySetName);
0602             if (typesFeatures != null) {
0603               for (String typeName : typesFeatures.keySet()) {
0604                 if (typeNames.contains(typeName)) {
0605                   if (featureCheck.isSelected() && !firstLoop) {
0606                     features.retainAll(typesFeatures.get(typeName));
0607                   else {
0608                     features.addAll(typesFeatures.get(typeName));
0609                   }
0610                 }
0611               }
0612             else if (featureCheck.isSelected()) {
0613               // empty type no features to display
0614               break;
0615             }
0616             typesFeatures = sets.get(responseSetName.equals("[Default set]"?
0617               "" : responseSetName);
0618             if (typesFeatures != null) {
0619               for (String typeName : typesFeatures.keySet()) {
0620                 if (typeNames.contains(typeName)) {
0621                   if (featureCheck.isSelected()) {
0622                     features.retainAll(typesFeatures.get(typeName));
0623                   else {
0624                     features.addAll(typesFeatures.get(typeName));
0625                   }
0626                 }
0627               }
0628             else if (featureCheck.isSelected()) {
0629               break;
0630             }
0631             firstLoop = false;
0632           }
0633         }
0634         featureList.setModel(new ExtendedListModel(features.toArray()));
0635         if (features.size() 0) {
0636           for (Object value : featuresSelected) {
0637             // put back the selection if possible
0638             int index = featureList.getNextMatch(
0639               (Stringvalue, 0, Position.Bias.Forward);
0640             if (index != -1) {
0641               featureList.setSelectedIndex(index);
0642             }
0643           }
0644         }
0645         featuresSelected = null;
0646         typeList.setEnabled(true);
0647         typeCheck.setEnabled(true);
0648       }
0649     });
0650 
0651     // when type list selection change
0652     featureList.addListSelectionListener(new ListSelectionListener() {
0653       @Override
0654       public void valueChanged(ListSelectionEvent e) {
0655         if (measuresType == CLASSIFICATION_MEASURES) {
0656           if (typeList.getSelectedIndices().length == 1
0657            && featureList.getSelectedIndices().length == 1) {
0658             compareAction.setEnabled(true);
0659             compareAction.putValue(Action.SHORT_DESCRIPTION,
0660               "Compare annotations between sets A and B");
0661           else {
0662             compareAction.setEnabled(false);
0663             compareAction.putValue(Action.SHORT_DESCRIPTION,
0664               "You must select exactly one type and one feature");
0665           }
0666         }
0667       }
0668     });
0669 
0670     // when the measure tab selection change
0671     measureTabbedPane.addChangeListener(new ChangeListener() {
0672       @Override
0673       public void stateChanged(ChangeEvent e) {
0674         JTabbedPane tabbedPane = (JTabbedPanee.getSource();
0675         int selectedTab = tabbedPane.getSelectedIndex();
0676         tableTabbedPane.removeAll();
0677         openDocumentAction.setEnabled(false);
0678         openAnnotationDiffAction.setEnabled(false);
0679         if (optionsButton.isSelected()) {
0680           optionsButton.doClick()// hide the options panel if shown
0681         }
0682         if (tabbedPane.getTitleAt(selectedTab).equals("F-Score")) {
0683           measuresType = FSCORE_MEASURES;
0684           compareAction.setEnabled(keySetName != null
0685                            && responseSetName != null);
0686           compareAction.putValue(Action.SHORT_DESCRIPTION,
0687             "Compare annotations between sets A and B");
0688           tableTabbedPane.addTab("Corpus statistics", null,
0689             new JScrollPane(annotationTable),
0690             "Compare each annotation type for the whole corpus");
0691           tableTabbedPane.addTab("Document statistics", null,
0692             new JScrollPane(documentTable),
0693             "Compare each documents in the corpus with theirs annotations");
0694         else {
0695           measuresType = CLASSIFICATION_MEASURES;
0696           featureList.getListSelectionListeners()[0].valueChanged(null);
0697           tableTabbedPane.addTab("Document statistics", null,
0698             new JScrollPane(document2Table),
0699             "Compare each documents in the corpus with theirs annotations");
0700           tableTabbedPane.addTab("Confusion Matrices", null,
0701             new JScrollPane(confusionTable)"Describe how annotations in" +
0702               " one set are classified in the other and vice versa.");
0703         }
0704       }
0705     });
0706 
0707     // enable/disable toolbar icons according to the document table selection
0708     documentTable.getSelectionModel().addListSelectionListener(
0709       new ListSelectionListener() {
0710         @Override
0711         public void valueChanged(ListSelectionEvent e) {
0712           if (e.getValueIsAdjusting()) { return}
0713           boolean enabled = documentTable.getSelectedRow() != -1
0714             && !((String)documentTableModel.getValueAt(
0715             documentTable.getSelectedRow()0)).endsWith("summary");
0716           openDocumentAction.setEnabled(enabled);
0717           openAnnotationDiffAction.setEnabled(enabled);
0718         }
0719       }
0720     );
0721 
0722     // enable/disable toolbar icons according to the document 2 table selection
0723     document2Table.getSelectionModel().addListSelectionListener(
0724       new ListSelectionListener() {
0725         @Override
0726         public void valueChanged(ListSelectionEvent e) {
0727           if (e.getValueIsAdjusting()) { return}
0728           boolean enabled = document2Table.getSelectedRow() != -1
0729             && !((String)document2TableModel.getValueAt(
0730               document2Table.getSelectedRow(),
0731               0)).endsWith("summary");
0732           openDocumentAction.setEnabled(enabled);
0733           openAnnotationDiffAction.setEnabled(enabled);
0734         }
0735       }
0736     );
0737 
0738     // double click on a document loads it in the document editor
0739     documentTable.addMouseListener(new MouseAdapter() {
0740       @Override
0741       public void mouseClicked(MouseEvent e) {
0742         if (!e.isPopupTrigger()
0743           && e.getClickCount() == 2
0744           && openDocumentAction.isEnabled()) {
0745           openDocumentAction.actionPerformed(null);
0746         }
0747       }
0748     });
0749 
0750     // double click on a document loads it in the document editor
0751     document2Table.addMouseListener(new MouseAdapter() {
0752       @Override
0753       public void mouseClicked(MouseEvent e) {
0754         if (!e.isPopupTrigger()
0755           && e.getClickCount() == 2
0756           && openDocumentAction.isEnabled()) {
0757           openDocumentAction.actionPerformed(null);
0758         }
0759       }
0760     });
0761 
0762     InputMap inputMap = getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW);
0763     ActionMap actionMap = getActionMap();
0764     inputMap.put(KeyStroke.getKeyStroke("F1")"help");
0765     actionMap.put("help"new HelpAction());
0766   }
0767 
0768   /**
0769    * Create a table header tool tips from the column name.
0770    @param columnName name used for creating the tooltip
0771    @return tooltip value
0772    */
0773   protected String createToolTipFromColumnName(String columnName) {
0774     String tooltip;
0775     if (columnName.equals("Document")
0776      || columnName.equals("Annotation")) {
0777       tooltip = null;
0778     else if (columnName.equals("Match")) {
0779       tooltip = "aka Correct";
0780     else if (columnName.equals("Only A")) {
0781       tooltip = "aka Missing";
0782     else if (columnName.equals("Only B")) {
0783       tooltip = "aka Spurious";
0784     else if (columnName.equals("Overlap")) {
0785       tooltip = "aka Partial";
0786     else if (columnName.equals("Rec.B/A")) {
0787       tooltip = "Recall for B relative to A";
0788     else if (columnName.equals("Prec.B/A")) {
0789       tooltip = "Precision for B relative to A";
0790     else {
0791       tooltip = columnName
0792         .replaceFirst("s.""score strict")
0793         .replaceFirst("l.""score lenient")
0794         .replaceFirst("a.""score average")
0795         .replaceFirst("B."" BDM");
0796     }
0797     return tooltip;
0798   }
0799 
0800   protected static class ExtendedListModel extends DefaultListModel {
0801     public ExtendedListModel() {
0802       super();
0803     }
0804     public ExtendedListModel(Object[] elements) {
0805       super();
0806       for (Object element : elements) {
0807         super.addElement(element);
0808       }
0809     }
0810   }
0811 
0812   protected static class ToggleSelectionModel extends DefaultListSelectionModel {
0813     boolean gestureStarted = false;
0814     @Override
0815     public void setSelectionInterval(int index0, int index1) {
0816       if (isSelectedIndex(index0&& !gestureStarted) {
0817         super.removeSelectionInterval(index0, index1);
0818       else {
0819         super.addSelectionInterval(index0, index1);
0820       }
0821       gestureStarted = true;
0822     }
0823     @Override
0824     public void setValueIsAdjusting(boolean isAdjusting) {
0825       if (!isAdjusting) {
0826         gestureStarted = false;
0827       }
0828     }
0829   }
0830 
0831   /**
0832    * Add a suffix A and B for the first and second selected item.
0833    * Allows only 2 items to be selected.
0834    */
0835   protected static class ToggleSelectionABModel extends DefaultListSelectionModel {
0836     public ToggleSelectionABModel(JList list) {
0837       this.list = list;
0838     }
0839     @Override
0840     public void setSelectionInterval(int index0, int index1) {
0841       ExtendedListModel model = (ExtendedListModellist.getModel();
0842       String value = (Stringmodel.getElementAt(index0);
0843       if (value.endsWith(" (A)"|| value.endsWith(" (B)")) {
0844         // if ends with ' (A)' or ' (B)' then remove the suffix
0845         model.removeElementAt(index0);
0846         model.insertElementAt(value.substring(0,
0847           value.length() " (A)".length()), index0);
0848         if (value.endsWith(" (A)")) {
0849           selectedValueA = null;
0850         else {
0851           selectedValueB = null;
0852         }
0853         removeSelectionInterval(index0, index1);
0854       else {
0855         // suffix with ' (A)' or ' (B)' if not already existing
0856         if (selectedValueA == null) {
0857           model.removeElementAt(index0);
0858           model.insertElementAt(value + " (A)", index0);
0859           selectedValueA = value;
0860           addSelectionInterval(index0, index1);
0861         else if (selectedValueB == null) {
0862           model.removeElementAt(index0);
0863           model.insertElementAt(value + " (B)", index0);
0864           selectedValueB = value;
0865           addSelectionInterval(index0, index1);
0866         }
0867       }
0868     }
0869     @Override
0870     public void clearSelection() {
0871       selectedValueA = null;
0872       selectedValueB = null;
0873       super.clearSelection();
0874     }
0875     public String getSelectedValueA() {
0876       return selectedValueA;
0877     }
0878     public String getSelectedValueB() {
0879       return selectedValueB;
0880     }
0881     JList list;
0882     String selectedValueA, selectedValueB;
0883   }
0884 
0885   @Override
0886   public void cleanup(){
0887     super.cleanup();
0888     corpus = null;
0889   }
0890 
0891   @Override
0892   public void setTarget(Object target){
0893     if(corpus != null && corpus != target){
0894       //we already had a different corpus
0895       corpus.removeCorpusListener(this);
0896     }
0897     if(!(target instanceof Corpus)){
0898       throw new IllegalArgumentException(
0899         "This view can only be used with a GATE corpus!\n" +
0900         target.getClass().toString() " is not a GATE corpus!");
0901     }
0902     this.corpus = (Corpustarget;
0903     corpus.addCorpusListener(this);
0904 
0905     corpusChanged = true;
0906     if (!isShowing()) { return}
0907     if (timerTask != null) { timerTask.cancel()}
0908     Date timeToRun = new Date(System.currentTimeMillis() 2000);
0909     timerTask = new TimerTask() { @Override
0910     public void run() {
0911       readSetsTypesFeatures(0);
0912     }};
0913     timer.schedule(timerTask, timeToRun)// add a delay before updating
0914   }
0915 
0916   @Override
0917   public void documentAdded(final CorpusEvent e) {
0918     corpusChanged = true;
0919     if (!isShowing()) { return}
0920     if (timerTask != null) { timerTask.cancel()}
0921     Date timeToRun = new Date(System.currentTimeMillis() 2000);
0922     timerTask = new TimerTask() { @Override
0923     public void run() {
0924       readSetsTypesFeatures(0);
0925     }};
0926     timer.schedule(timerTask, timeToRun)// add a delay before updating
0927   }
0928 
0929   @Override
0930   public void documentRemoved(final CorpusEvent e) {
0931     corpusChanged = true;
0932     if (!isShowing()) { return}
0933     if (timerTask != null) { timerTask.cancel()}
0934     Date timeToRun = new Date(System.currentTimeMillis() 2000);
0935     timerTask = new TimerTask() { @Override
0936     public void run() {
0937       readSetsTypesFeatures(0);
0938     }};
0939     timer.schedule(timerTask, timeToRun)// add a delay before updating
0940   }
0941 
0942   /**
0943    * Update set lists.
0944    @param documentStart first document to read in the corpus,
0945    * the first document of the corpus is 0.
0946    */
0947   protected void readSetsTypesFeatures(final int documentStart) {
0948     if (!isShowing()) { return}
0949     corpusChanged = false;
0950     SwingUtilities.invokeLater(new Runnable(){ @Override
0951     public void run() {
0952       progressBar.setMaximum(corpus.size() 1);
0953       progressBar.setString("Read sets, types, features");
0954       reloadCacheAction.setEnabled(false);
0955     }});
0956     CorpusQualityAssurance.this.setCursor(
0957       Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
0958     Runnable runnable = new Runnable() { @Override
0959     public void run() {
0960     if (docsSetsTypesFeatures.size() != corpus.getDocumentNames().size()
0961     || !docsSetsTypesFeatures.keySet().containsAll(corpus.getDocumentNames())) {
0962       if (documentStart == 0) { docsSetsTypesFeatures.clear()}
0963       TreeMap<String, TreeMap<String, TreeSet<String>>> setsTypesFeatures;
0964       TreeMap<String, TreeSet<String>> typesFeatures;
0965       TreeSet<String> features;
0966       for (int i = documentStart; i < corpus.size(); i++) {
0967         // fill in the lists of document, set, type and feature names
0968         boolean documentWasLoaded = corpus.isDocumentLoaded(i);
0969         Document document = corpus.get(i);
0970         if (document != null && document.getAnnotationSetNames() != null) {
0971           setsTypesFeatures =
0972             new TreeMap<String, TreeMap<String, TreeSet<String>>>(collator);
0973           HashSet<String> setNames =
0974             new HashSet<String>(document.getAnnotationSetNames());
0975           setNames.add("");
0976           for (String set : setNames) {
0977             typesFeatures = new TreeMap<String, TreeSet<String>>(collator);
0978             AnnotationSet annotations = document.getAnnotations(set);
0979             for (String type : annotations.getAllTypes()) {
0980               features = new TreeSet<String>(collator);
0981               for (Annotation annotation : annotations.get(type)) {
0982                 for (Object featureKey : annotation.getFeatures().keySet()) {
0983                   features.add((StringfeatureKey);
0984                 }
0985               }
0986               typesFeatures.put(type, features);
0987             }
0988             setsTypesFeatures.put(set, typesFeatures);
0989           }
0990           docsSetsTypesFeatures.put(document.getName(), setsTypesFeatures);
0991         }
0992         if (!documentWasLoaded) {
0993           corpus.unloadDocument(document);
0994           Factory.deleteResource(document);
0995         }
0996         final int progressValue = i + 1;
0997         SwingUtilities.invokeLater(new Runnable(){ @Override
0998         public void run() {
0999           progressBar.setValue(progressValue);
1000           if ((progressValue+1== 0) {
1001             // update the set list every 5 documents read
1002             updateSetList();
1003           }
1004         }});
1005         if (Thread.interrupted()) { return}
1006       }
1007     }
1008     updateSetList();
1009     SwingUtilities.invokeLater(new Runnable(){ @Override
1010     public void run(){
1011       progressBar.setValue(progressBar.getMinimum());
1012       progressBar.setString("");
1013       CorpusQualityAssurance.this.setCursor(
1014         Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
1015       reloadCacheAction.setEnabled(true);
1016     }});
1017     }};
1018     readSetsTypesFeaturesThread = new Thread(runnable, "readSetsTypesFeatures");
1019     readSetsTypesFeaturesThread.setPriority(Thread.MIN_PRIORITY);
1020     readSetsTypesFeaturesThread.start();
1021   }
1022 
1023   protected void updateSetList() {
1024     final TreeSet<String> setsNames = new TreeSet<String>(collator);
1025     Set<String> sets;
1026     boolean firstLoop = true// needed for retainAll to work
1027     synchronized(docsSetsTypesFeatures) {
1028       for (String document : docsSetsTypesFeatures.keySet()) {
1029         // get the list of set names
1030         sets = docsSetsTypesFeatures.get(document).keySet();
1031         if (!sets.isEmpty()) {
1032           if (setCheck.isSelected() && !firstLoop) {
1033             setsNames.retainAll(sets);
1034           else {
1035             setsNames.addAll(sets);
1036           }
1037         else if (setCheck.isSelected()) {
1038           break;
1039         }
1040         firstLoop = false;
1041       }
1042     }
1043     SwingUtilities.invokeLater(new Runnable(){ @Override
1044     public void run() {
1045       // update the UI lists of sets
1046       setsNames.remove("");
1047       setsNames.add("[Default set]");
1048       String keySetNamePrevious = keySetName;
1049       String responseSetNamePrevious = responseSetName;
1050       setList.setModel(new ExtendedListModel(setsNames.toArray()));
1051       if (setsNames.size() 0) {
1052         if (keySetNamePrevious != null) {
1053           // put back the selection if possible
1054           int index = setList.getNextMatch(
1055             keySetNamePrevious, 0, Position.Bias.Forward);
1056           if (index != -1) {
1057             setList.setSelectedIndex(index);
1058           }
1059         }
1060         if (responseSetNamePrevious != null) {
1061           // put back the selection if possible
1062           int index = setList.getNextMatch(
1063             responseSetNamePrevious, 0, Position.Bias.Forward);
1064           if (index != -1) {
1065             setList.setSelectedIndex(index);
1066           }
1067         }
1068       }
1069     }});
1070   }
1071 
1072   protected void compareAnnotation() {
1073     int progressValuePrevious = -1;
1074     if (readSetsTypesFeaturesThread != null
1075      && readSetsTypesFeaturesThread.isAlive()) {
1076       // stop the thread that reads the sets, types and features
1077       progressValuePrevious = progressBar.getValue();
1078       readSetsTypesFeaturesThread.interrupt();
1079     }
1080     SwingUtilities.invokeLater(new Runnable() { @Override
1081     public void run() {
1082       progressBar.setMaximum(corpus.size() 1);
1083       progressBar.setString("Compare annotations");
1084       setList.setEnabled(false);
1085       setCheck.setEnabled(false);
1086       typeList.setEnabled(false);
1087       typeCheck.setEnabled(false);
1088       featureList.setEnabled(false);
1089       featureCheck.setEnabled(false);
1090       optionsButton.setEnabled(false);
1091       measureTabbedPane.setEnabled(false);
1092       measureList.setEnabled(false);
1093       exportToHtmlAction.setEnabled(false);
1094       reloadCacheAction.setEnabled(false);
1095     }});
1096 
1097     boolean useBdm = false;
1098     if (measuresType == FSCORE_MEASURES) {
1099       differsByDocThenType.clear();
1100       documentNames.clear();
1101       for (Object measure : measureList.getSelectedValues()) {
1102         if (((Stringmeasure).contains("BDM")) { useBdm = truebreak}
1103       }
1104     }
1105     List<ClassificationMeasures> classificationMeasuresList =
1106       new ArrayList<ClassificationMeasures>();
1107     List<OntologyMeasures> documentOntologyMeasuresList =
1108       new ArrayList<OntologyMeasures>();
1109     List<OntologyMeasures> annotationOntologyMeasuresList =
1110       new ArrayList<OntologyMeasures>();
1111 
1112     // for each document
1113     for (int row = 0; row < corpus.size(); row++) {
1114       boolean documentWasLoaded = corpus.isDocumentLoaded(row);
1115       Document document = corpus.get(row);
1116       documentNames.add(document.getName());
1117       Set<Annotation> keys = new HashSet<Annotation>();
1118       Set<Annotation> responses = new HashSet<Annotation>();
1119       // get annotations from selected annotation sets
1120       if (keySetName.equals("[Default set]")) {
1121         keys = document.getAnnotations();
1122       else if (document.getAnnotationSetNames() != null
1123       && document.getAnnotationSetNames().contains(keySetName)) {
1124         keys = document.getAnnotations(keySetName);
1125       }
1126       if (responseSetName.equals("[Default set]")) {
1127         responses = document.getAnnotations();
1128       else if (document.getAnnotationSetNames() != null
1129       && document.getAnnotationSetNames()
1130         .contains(responseSetName)) {
1131         responses = document.getAnnotations(responseSetName);
1132       }
1133       if (!documentWasLoaded) { // in case of datastore
1134         corpus.unloadDocument(document);
1135         Factory.deleteResource(document);
1136       }
1137 
1138       // add data to the fscore document table
1139       if (measuresType == FSCORE_MEASURES) {
1140         types.clear();
1141         for (Object type : typeList.getSelectedValues()) {
1142           types.add((Stringtype);
1143         }
1144         if (typeList.isSelectionEmpty()) {
1145           for (int i = 0; i < typeList.getModel().getSize(); i++) {
1146             types.add((StringtypeList.getModel().getElementAt(i));
1147           }
1148         }
1149         Set<String> featureSet = new HashSet<String>();
1150         for (Object feature : featureList.getSelectedValues()) {
1151           featureSet.add((Stringfeature);
1152         }
1153         HashMap<String, AnnotationDiffer> differsByType =
1154           new HashMap<String, AnnotationDiffer>();
1155         AnnotationDiffer differ;
1156         Set<Annotation> keysIter = new HashSet<Annotation>();
1157         Set<Annotation> responsesIter = new HashSet<Annotation>();
1158         for (String type : types) {
1159           if (!keys.isEmpty() && !types.isEmpty()) {
1160             keysIter = ((AnnotationSet)keys).get(type);
1161           }
1162           if (!responses.isEmpty() && !types.isEmpty()) {
1163             responsesIter = ((AnnotationSet)responses).get(type);
1164           }
1165           differ = new AnnotationDiffer();
1166           differ.setSignificantFeaturesSet(featureSet);
1167           differ.calculateDiff(keysIter, responsesIter)// compare
1168           differsByType.put(type, differ);
1169         }
1170         differsByDocThenType.add(differsByType);
1171         differ = new AnnotationDiffer(differsByType.values());
1172         List<String> measuresRow;
1173         if (useBdm) {
1174           OntologyMeasures ontologyMeasures = new OntologyMeasures();
1175           ontologyMeasures.setBdmFile(bdmFileUrl);
1176           ontologyMeasures.calculateBdm(differsByType.values());
1177           documentOntologyMeasuresList.add(ontologyMeasures);
1178           measuresRow = ontologyMeasures.getMeasuresRow(
1179             measureList.getSelectedValues(),
1180             documentNames.get(documentNames.size()-1));
1181         else {
1182           measuresRow = differ.getMeasuresRow(measureList.getSelectedValues(),
1183             documentNames.get(documentNames.size()-1));
1184         }
1185         documentTableModel.addRow(measuresRow.toArray());
1186 
1187         // add data to the classification document table
1188       else if (measuresType == CLASSIFICATION_MEASURES
1189              && !keys.isEmpty() && !responses.isEmpty()) {
1190         ClassificationMeasures classificationMeasures =
1191           new ClassificationMeasures();
1192         classificationMeasures.calculateConfusionMatrix(
1193           (AnnotationSetkeys, (AnnotationSetresponses,
1194           (StringtypeList.getSelectedValue(),
1195           (StringfeatureList.getSelectedValue(),
1196           verboseOptionCheckBox.isSelected());
1197         classificationMeasuresList.add(classificationMeasures);
1198         List<String> measuresRow = classificationMeasures.getMeasuresRow(
1199           measure2List.getSelectedValues(),
1200           documentNames.get(documentNames.size()-1));
1201         document2TableModel.addRow(measuresRow.toArray());
1202         List<List<String>> matrix = classificationMeasures
1203           .getConfusionMatrix(documentNames.get(documentNames.size()-1));
1204         for (List<String> matrixRow : matrix) {
1205           while (confusionTableModel.getColumnCount() < matrix.size()) {
1206             confusionTableModel.addColumn(" ");
1207           }
1208           confusionTableModel.addRow(matrixRow.toArray());
1209         }
1210       }
1211       final int progressValue = row + 1;
1212       SwingUtilities.invokeLater(new Runnable(){ @Override
1213       public void run() {
1214         progressBar.setValue(progressValue);
1215       }});
1216     // for (int row = 0; row < corpus.size(); row++)
1217 
1218     // add data to the fscore annotation table
1219     if (measuresType == FSCORE_MEASURES) {
1220       for (String type : types) {
1221         ArrayList<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>();
1222         for (HashMap<String, AnnotationDiffer> differsByType :
1223               differsByDocThenType) {
1224           differs.add(differsByType.get(type));
1225         }
1226         List<String> measuresRow;
1227         if (useBdm) {
1228           OntologyMeasures ontologyMeasures = new OntologyMeasures();
1229           ontologyMeasures.setBdmFile(bdmFileUrl);
1230           ontologyMeasures.calculateBdm(differs);
1231           annotationOntologyMeasuresList.add(ontologyMeasures);
1232           measuresRow = ontologyMeasures.getMeasuresRow(
1233             measureList.getSelectedValues(), type);
1234         else {
1235           AnnotationDiffer differ = new AnnotationDiffer(differs);
1236           measuresRow = differ.getMeasuresRow(
1237             measureList.getSelectedValues(), type);
1238         }
1239         annotationTableModel.addRow(measuresRow.toArray());
1240       }
1241     }
1242 
1243     // add summary rows to the fscore tables
1244     if (measuresType == FSCORE_MEASURES) {
1245       if (useBdm) {
1246         OntologyMeasures ontologyMeasures =
1247           new OntologyMeasures(documentOntologyMeasuresList);
1248         printSummary(ontologyMeasures, documentTableModel, 5,
1249           documentTableModel.getRowCount(), measureList.getSelectedValues());
1250         ontologyMeasures = new OntologyMeasures(annotationOntologyMeasuresList);
1251         printSummary(ontologyMeasures, annotationTableModel, 5,
1252           annotationTableModel.getRowCount(), measureList.getSelectedValues());
1253       else {
1254         List<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>();
1255         for (Map<String, AnnotationDiffer> differsByType :
1256               differsByDocThenType) {
1257           differs.addAll(differsByType.values());
1258         }
1259         AnnotationDiffer differ = new AnnotationDiffer(differs);
1260         printSummary(differ, documentTableModel, 5,
1261           documentTableModel.getRowCount(), measureList.getSelectedValues());
1262         printSummary(differ, annotationTableModel, 5,
1263           annotationTableModel.getRowCount(), measureList.getSelectedValues());
1264       }
1265 
1266       // add summary rows to the classification tables
1267     else if (measuresType == CLASSIFICATION_MEASURES) {
1268       ClassificationMeasures classificationMeasures =
1269         new ClassificationMeasures(classificationMeasuresList);
1270       printSummary(classificationMeasures, document2TableModel, 3,
1271         document2TableModel.getRowCount(), measure2List.getSelectedValues());
1272       List<List<String>> matrix = classificationMeasures
1273         .getConfusionMatrix("Whole corpus");
1274       int insertionRow = 0;
1275       for (List<String> row : matrix) {
1276         while (confusionTableModel.getColumnCount() < matrix.size()) {
1277           confusionTableModel.addColumn(" ");
1278         }
1279         confusionTableModel.insertRow(insertionRow++, row.toArray());
1280       }
1281     }
1282 
1283     SwingUtilities.invokeLater(new Runnable(){ @Override
1284     public void run(){
1285       progressBar.setValue(progressBar.getMinimum());
1286       progressBar.setString("");
1287       setList.setEnabled(true);
1288       setCheck.setEnabled(true);
1289       typeList.setEnabled(true);
1290       typeCheck.setEnabled(true);
1291       featureList.setEnabled(true);
1292       featureCheck.setEnabled(true);
1293       optionsButton.setEnabled(true);
1294       measureTabbedPane.setEnabled(true);
1295       measureList.setEnabled(true);
1296       exportToHtmlAction.setEnabled(true);
1297       reloadCacheAction.setEnabled(true);
1298     }});
1299     if (progressValuePrevious > -1) {
1300       // restart the thread where it was interrupted
1301       readSetsTypesFeatures(progressValuePrevious);
1302     }
1303   }
1304 
1305   protected void printSummary(Object measureObject,
1306                               DefaultTableModel tableModel, int columnGroupSize,
1307                               int insertionRow, Object[] measures) {
1308     AnnotationDiffer differ = null;
1309     ClassificationMeasures classificationMeasures = null;
1310     OntologyMeasures ontologyMeasures = null;
1311     if (measureObject instanceof AnnotationDiffer) {
1312       differ = (AnnotationDiffermeasureObject;
1313     else if (measureObject instanceof ClassificationMeasures) {
1314       classificationMeasures = (ClassificationMeasuresmeasureObject;
1315     else if (measureObject instanceof OntologyMeasures) {
1316       ontologyMeasures = (OntologyMeasuresmeasureObject;
1317     }
1318     NumberFormat f = NumberFormat.getInstance(Locale.ENGLISH);
1319     f.setMaximumFractionDigits(4);
1320     f.setMinimumFractionDigits(4);
1321     f.setRoundingMode(RoundingMode.HALF_UP);
1322     List<Object> values = new ArrayList<Object>();
1323 
1324     // average measures by document
1325     values.add("Macro summary");
1326     for (int col = 1; col < tableModel.getColumnCount(); col++) {
1327       if (col < columnGroupSize) {
1328         values.add("");
1329       else {
1330         float sumF = 0;
1331         for (int row = 0; row < tableModel.getRowCount(); row++) {
1332           try {
1333             sumF += Float.parseFloat((StringtableModel.getValueAt(row, col));
1334           catch(NumberFormatException e) {
1335             // do nothing
1336           }
1337         }
1338         values.add(f.format(sumF / tableModel.getRowCount()));
1339       }
1340     }
1341     tableModel.insertRow(insertionRow, values.toArray());
1342 
1343     // sum counts and recalculate measures like the corpus is one document
1344     values.clear();
1345     values.add("Micro summary");
1346     for (int col = 1; col < columnGroupSize; col++) {
1347       int sum = 0;
1348       for (int row = 0; row < tableModel.getRowCount()-1; row++) {
1349         try {
1350           sum += Integer.valueOf((StringtableModel.getValueAt(row, col));
1351         catch(NumberFormatException e) {
1352           // do nothing
1353         }
1354       }
1355       values.add(Integer.toString(sum));
1356     }
1357     if (measureObject instanceof OntologyMeasures) {
1358       List<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>(
1359         ontologyMeasures.getDifferByTypeMap().values());
1360       differ = new AnnotationDiffer(differs);
1361     }
1362     for (Object object : measures) {
1363       String measure = (Stringobject;
1364       int index = measure.indexOf('-');
1365       double beta = (index == -1?
1366         : Double.valueOf(measure.substring(1, index));
1367       if (measure.endsWith("strict")) {
1368         values.add(f.format(differ.getPrecisionStrict()));
1369         values.add(f.format(differ.getRecallStrict()));
1370         values.add(f.format(differ.getFMeasureStrict(beta)));
1371       else if (measure.endsWith("strict BDM")) {
1372         values.add(f.format(ontologyMeasures.getPrecisionStrictBdm()));
1373         values.add(f.format(ontologyMeasures.getRecallStrictBdm()));
1374         values.add(f.format(ontologyMeasures.getFMeasureStrictBdm(beta)));
1375       else if (measure.endsWith("lenient")) {
1376         values.add(f.format(differ.getPrecisionLenient()));
1377         values.add(f.format(differ.getRecallLenient()));
1378         values.add(f.format(differ.getFMeasureLenient(beta)));
1379       else if (measure.endsWith("lenient BDM")) {
1380         values.add(f.format(ontologyMeasures.getPrecisionLenientBdm()));
1381         values.add(f.format(ontologyMeasures.getRecallLenientBdm()));
1382         values.add(f.format(ontologyMeasures.getFMeasureLenientBdm(beta)));
1383       else if (measure.endsWith("average")) {
1384         values.add(f.format(differ.getPrecisionAverage()));
1385         values.add(f.format(differ.getRecallAverage()));
1386         values.add(f.format(differ.getFMeasureAverage(beta)));
1387       else if (measure.endsWith("average BDM")) {
1388         values.add(f.format(ontologyMeasures.getPrecisionAverageBdm()));
1389         values.add(f.format(ontologyMeasures.getRecallAverageBdm()));
1390         values.add(f.format(ontologyMeasures.getFMeasureAverageBdm(beta)));
1391       else if (measure.equals("Observed agreement")) {
1392         values.add(f.format(classificationMeasures.getObservedAgreement()));
1393       else if (measure.equals("Cohen's Kappa")) {
1394         float result = classificationMeasures.getKappaCohen();
1395         values.add(Float.isNaN(result"" : f.format(result));
1396       else if (measure.equals("Pi's Kappa")) {
1397         float result = classificationMeasures.getKappaPi();
1398         values.add(Float.isNaN(result"" : f.format(result));
1399       }
1400     }
1401     tableModel.insertRow(insertionRow + 1, values.toArray());
1402   }
1403 
1404   protected class SetBdmFileAction extends AbstractAction {
1405     public SetBdmFileAction() {
1406       super("Browse");
1407       putValue(SHORT_DESCRIPTION, "Choose a BDM file to compute BDM measures");
1408     }
1409     @Override
1410     public void actionPerformed(ActionEvent evt) {
1411       XJFileChooser fileChooser = MainFrame.getFileChooser();
1412       fileChooser.setAcceptAllFileFilterUsed(true);
1413       fileChooser.setDialogTitle("Choose a BDM file");
1414       fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
1415       fileChooser.setResource(
1416         CorpusQualityAssurance.class.getName() ".BDMfile");
1417       int res = fileChooser.showOpenDialog(CorpusQualityAssurance.this);
1418       if (res != JFileChooser.APPROVE_OPTION) { return}
1419       try {
1420         bdmFileUrl = fileChooser.getSelectedFile().toURI().toURL();
1421       catch (MalformedURLException e) {
1422         e.printStackTrace();
1423       }
1424     }
1425   }
1426 
1427   /**
1428    * Update document table.
1429    */
1430   protected class CompareAction extends AbstractAction {
1431     public CompareAction() {
1432       super("Compare");
1433       putValue(SHORT_DESCRIPTION, "Compare annotations between sets A and B");
1434       putValue(MNEMONIC_KEY, KeyEvent.VK_ENTER);
1435       putValue(SMALL_ICON, MainFrame.getIcon("crystal-clear-action-run"));
1436     }
1437     @Override
1438     public void actionPerformed(ActionEvent evt) {
1439       boolean useBdm = false;
1440       for (Object measure : measureList.getSelectedValues()) {
1441         if (((Stringmeasure).contains("BDM")) { useBdm = truebreak}
1442       }
1443       if (useBdm && measuresType == FSCORE_MEASURES && bdmFileUrl == null) {
1444         new SetBdmFileAction().actionPerformed(null);
1445         if (bdmFileUrl == null) { return}
1446       }
1447 
1448       CorpusQualityAssurance.this.setCursor(
1449         Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
1450       setEnabled(false);
1451 
1452       Runnable runnable = new Runnable() { @Override
1453       public void run() {
1454       if (measuresType == FSCORE_MEASURES) {
1455         documentTableModel = new DefaultTableModel();
1456         annotationTableModel = new DefaultTableModel();
1457         documentTableModel.addColumn("Document");
1458         annotationTableModel.addColumn("Annotation");
1459         documentTableModel.addColumn("Match");
1460         annotationTableModel.addColumn("Match");
1461         documentTableModel.addColumn("Only A");
1462         annotationTableModel.addColumn("Only A");
1463         documentTableModel.addColumn("Only B");
1464         annotationTableModel.addColumn("Only B");
1465         documentTableModel.addColumn("Overlap");
1466         annotationTableModel.addColumn("Overlap");
1467         for (Object measure : measureList.getSelectedValues()) {
1468           String measureString = ((Stringmeasure)
1469             .replaceFirst("score strict""s.")
1470             .replaceFirst("score lenient""l.")
1471             .replaceFirst("score average""a.")
1472             .replaceFirst(" BDM""B.");
1473           documentTableModel.addColumn("Prec.B/A");
1474           annotationTableModel.addColumn("Prec.B/A");
1475           documentTableModel.addColumn("Rec.B/A");
1476           annotationTableModel.addColumn("Rec.B/A");
1477           documentTableModel.addColumn(measureString);
1478           annotationTableModel.addColumn(measureString);
1479         }
1480         compareAnnotation()// do all the computation
1481         // update data
1482 
1483         SwingUtilities.invokeLater(new Runnable() { @Override
1484         public void run() {
1485           // redraw document table
1486           documentTable.setModel(documentTableModel);
1487           for (int col = 0; col < documentTable.getColumnCount(); col++) {
1488             documentTable.setComparator(col, doubleComparator);
1489           }
1490           documentTable.setComparator(0, totalComparator);
1491           documentTable.setSortedColumn(0);
1492           // redraw annotation table
1493           annotationTable.setModel(annotationTableModel);
1494           for (int col = 0; col < annotationTable.getColumnCount(); col++) {
1495             annotationTable.setComparator(col, doubleComparator);
1496           }
1497           annotationTable.setComparator(0, totalComparator);
1498           annotationTable.setSortedColumn(0);
1499           CorpusQualityAssurance.this.setCursor(
1500             Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
1501           setEnabled(true);
1502         }});
1503 
1504       else if (measuresType == CLASSIFICATION_MEASURES) {
1505         document2TableModel = new DefaultTableModel();
1506         document2TableModel.addColumn("Document");
1507         document2TableModel.addColumn("Agreed");
1508         document2TableModel.addColumn("Total");
1509         for (Object measure : measure2List.getSelectedValues()) {
1510           document2TableModel.addColumn(measure);
1511         }
1512         confusionTableModel = new DefaultTableModel();
1513         compareAnnotation()// do all the computation
1514         SwingUtilities.invokeLater(new Runnable() { @Override
1515         public void run() {
1516           document2Table.setSortable(false);
1517           document2Table.setModel(document2TableModel);
1518           document2Table.setComparator(0, totalComparator);
1519           document2Table.setComparator(1, doubleComparator);
1520           document2Table.setSortedColumn(0);
1521           document2Table.setSortable(true);
1522           confusionTable.setModel(confusionTableModel);
1523           CorpusQualityAssurance.this.setCursor(
1524             Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
1525           setEnabled(true);
1526         }});
1527       }
1528       }};
1529       Thread thread = new Thread(runnable,  "CompareAction");
1530       thread.setPriority(Thread.MIN_PRIORITY);
1531       thread.start();
1532     }
1533   }
1534 
1535   class OpenDocumentAction extends AbstractAction{
1536     public OpenDocumentAction(){
1537       super("Open documents", MainFrame.getIcon("document"));
1538       putValue(SHORT_DESCRIPTION,
1539         "Opens document for the selected row in a document editor");
1540       putValue(MNEMONIC_KEY, KeyEvent.VK_UP);
1541     }
1542     @Override
1543     public void actionPerformed(ActionEvent e){
1544       final Document document = corpus.get(measuresType == FSCORE_MEASURES ?
1545         documentTable.rowViewToModel(documentTable.getSelectedRow())
1546       : document2Table.rowViewToModel(document2Table.getSelectedRow()));
1547       SwingUtilities.invokeLaternew Runnable() { @Override
1548       public void run() {
1549         MainFrame.getInstance().select(document);
1550       }});
1551     }
1552   }
1553 
1554   class OpenAnnotationDiffAction extends AbstractAction{
1555     public OpenAnnotationDiffAction(){
1556       super("Open annotation diff", MainFrame.getIcon("annDiff"));
1557       putValue(SHORT_DESCRIPTION,
1558         "Opens annotation diff for the selected row in the document table");
1559       putValue(MNEMONIC_KEY, KeyEvent.VK_RIGHT);
1560     }
1561     @Override
1562     public void actionPerformed(ActionEvent e){
1563       Document document = corpus.get(measuresType == FSCORE_MEASURES ?
1564         documentTable.rowViewToModel(documentTable.getSelectedRow())
1565       : document2Table.rowViewToModel(document2Table.getSelectedRow()));
1566       String documentName = document.getName();
1567       String annotationType = (StringtypeList.getSelectedValue();
1568       Set<String> featureSet = new HashSet<String>();
1569       for (Object feature : featureList.getSelectedValues()) {
1570         featureSet.add((Stringfeature);
1571       }
1572       AnnotationDiffGUI frame = new AnnotationDiffGUI("Annotation Difference",
1573         documentName, documentName, keySetName,
1574         responseSetName, annotationType, featureSet);
1575       frame.pack();
1576       frame.setLocationRelativeTo(MainFrame.getInstance());
1577       frame.setVisible(true);
1578     }
1579   }
1580 
1581   protected class ExportToHtmlAction extends AbstractAction{
1582     public ExportToHtmlAction(){
1583       super("Export to HTML");
1584       putValue(SHORT_DESCRIPTION, "Export the tables to HTML");
1585       putValue(SMALL_ICON,
1586         MainFrame.getIcon("crystal-clear-app-download-manager"));
1587     }
1588     @Override
1589     public void actionPerformed(ActionEvent evt){
1590       XJFileChooser fileChooser = MainFrame.getFileChooser();
1591       fileChooser.setAcceptAllFileFilterUsed(true);
1592       fileChooser.setDialogTitle("Choose a file where to export the tables");
1593       fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
1594       ExtensionFileFilter filter = new ExtensionFileFilter("HTML files","html");
1595       fileChooser.addChoosableFileFilter(filter);
1596       String title = corpus.getName();
1597         title += "_" + keySetName;
1598         title += "-" + responseSetName;
1599       fileChooser.setFileName(title + ".html");
1600       fileChooser.setResource(CorpusQualityAssurance.class.getName());
1601       int res = fileChooser.showSaveDialog(CorpusQualityAssurance.this);
1602       if (res != JFileChooser.APPROVE_OPTION) { return}
1603 
1604       File saveFile = fileChooser.getSelectedFile();
1605       Writer fw = null;
1606       try{
1607         fw = new BufferedWriter(new FileWriter(saveFile));
1608 
1609         // Header, Title
1610         fw.write(BEGINHTML + nl);
1611         fw.write(BEGINHEAD);
1612         fw.write(title);
1613         fw.write(ENDHEAD + nl);
1614         fw.write("<H1>Corpus Quality Assurance</H1>" + nl);
1615         fw.write("<P>Corpus: " + corpus.getName() "<BR>" + nl);
1616         fw.write("Key set: " + keySetName + "<BR>" + nl);
1617         fw.write("Response set: " + responseSetName + "<BR>" + nl);
1618         fw.write("Types: "
1619           + Strings.toString(typeList.getSelectedValues()) "<BR>" + nl);
1620         fw.write("Features: "
1621           + Strings.toString(featureList.getSelectedValues()) "</P>" + nl);
1622         fw.write("<P>&nbsp;</P>" + nl);
1623 
1624         ArrayList<JTable> tablesToExport = new ArrayList<JTable>();
1625         tablesToExport.add(annotationTable);
1626         tablesToExport.add(documentTable);
1627         tablesToExport.add(document2Table);
1628         tablesToExport.add(confusionTable);
1629         for (JTable table : tablesToExport) {
1630           fw.write(BEGINTABLE + nl + "<TR>" + nl);
1631           for(int col = 0; col < table.getColumnCount(); col++){
1632             fw.write("<TH align=\"left\">"
1633               + table.getColumnName(col"</TH>" + nl);
1634           }
1635           fw.write("</TR>" + nl);
1636           for(int row = 0; row < table.getRowCount(); row ++){
1637             fw.write("<TR>" + nl);
1638             for(int col = 0; col < table.getColumnCount(); col++){
1639               String value = (Stringtable.getValueAt(row, col);
1640               if (value == null) { value = ""}
1641               fw.write("<TD>" + value  + "</TD>" + nl);
1642             }
1643             fw.write("</TR>" + nl);
1644           }
1645           fw.write(ENDTABLE + nl);
1646           fw.write("<P>&nbsp;</P>" + nl);
1647         }
1648 
1649         fw.write(ENDHTML + nl);
1650         fw.flush();
1651 
1652       catch(IOException ioe){
1653         JOptionPane.showMessageDialog(CorpusQualityAssurance.this,
1654           ioe.toString()"GATE", JOptionPane.ERROR_MESSAGE);
1655         ioe.printStackTrace();
1656 
1657       finally {
1658         if (fw != null) {
1659           try {
1660             fw.close();
1661           catch (IOException e) {
1662             e.printStackTrace();
1663           }
1664         }
1665       }
1666     }
1667 
1668     final String nl = Strings.getNl();
1669     static final String BEGINHTML =
1670       "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" +
1671       "<html>";
1672     static final String ENDHTML = "</body></html>";
1673     static final String BEGINHEAD = "<head>" +
1674       "<meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\">"
1675       "<title>";
1676     static final String ENDHEAD = "</title></head><body>";
1677     static final String BEGINTABLE = "<table cellpadding=\"0\" border=\"1\">";
1678     static final String ENDTABLE = "</table>";
1679   }
1680 
1681   class ReloadCacheAction extends AbstractAction{
1682     public ReloadCacheAction(){
1683       super("Reload cache", MainFrame.getIcon("crystal-clear-action-reload"));
1684       putValue(SHORT_DESCRIPTION,
1685         "Reload cache for set, type and feature names list");
1686     }
1687     @Override
1688     public void actionPerformed(ActionEvent e){
1689       docsSetsTypesFeatures.clear();
1690       readSetsTypesFeatures(0);
1691     }
1692   }
1693 
1694   protected class HelpAction extends AbstractAction {
1695     public HelpAction() {
1696       super();
1697       putValue(SHORT_DESCRIPTION, "User guide for this tool");
1698       putValue(SMALL_ICON, MainFrame.getIcon("crystal-clear-action-info"));
1699       putValue(ACCELERATOR_KEY, KeyStroke.getKeyStroke("F1"));
1700     }
1701     @Override
1702     public void actionPerformed(ActionEvent e) {
1703       MainFrame.getInstance().showHelpFrame(
1704         "sec:eval:corpusqualityassurance",
1705         CorpusQualityAssurance.class.getName());
1706     }
1707   }
1708 
1709   // local variables
1710   protected Corpus corpus;
1711   protected boolean corpusChanged;
1712   protected TreeSet<String> types;
1713   /** cache for document*set*type*feature names */
1714   final protected Map<String, TreeMap<String, TreeMap<String, TreeSet<String>>>>
1715     docsSetsTypesFeatures = Collections.synchronizedMap(new LinkedHashMap
1716       <String, TreeMap<String, TreeMap<String, TreeSet<String>>>>());
1717   /** ordered by document as in the <code>corpus</code>
1718    *  then contains (annotation type * AnnotationDiffer) */
1719   protected ArrayList<HashMap<String, AnnotationDiffer>> differsByDocThenType =
1720     new ArrayList<HashMap<String, AnnotationDiffer>>();
1721   protected ArrayList<String> documentNames = new ArrayList<String>();
1722   protected String keySetName;
1723   protected String responseSetName;
1724   protected Object[] typesSelected;
1725   protected Object[] featuresSelected;
1726   protected Timer timer = new Timer("CorpusQualityAssurance"true);
1727   protected TimerTask timerTask;
1728   protected Thread readSetsTypesFeaturesThread;
1729   /** FSCORE_MEASURES or CLASSIFICATION_MEASURES */
1730   protected int measuresType;
1731   protected static final int FSCORE_MEASURES = 0;
1732   protected static final int CLASSIFICATION_MEASURES = 1;
1733   protected Collator collator;
1734   protected Comparator<String> doubleComparator;
1735   protected Comparator<String> totalComparator;
1736   protected OptionsMap userConfig = Gate.getUserConfig();
1737   protected URL bdmFileUrl;
1738 
1739   // user interface components
1740   protected XJTable documentTable;
1741   protected DefaultTableModel documentTableModel;
1742   protected XJTable annotationTable;
1743   protected DefaultTableModel annotationTableModel;
1744   protected XJTable document2Table;
1745   protected DefaultTableModel document2TableModel;
1746   protected XJTable confusionTable;
1747   protected DefaultTableModel confusionTableModel;
1748   protected JTabbedPane tableTabbedPane;
1749   protected JList setList;
1750   protected JList typeList;
1751   protected JList featureList;
1752   protected JToggleButton optionsButton;
1753   protected JTabbedPane measureTabbedPane;
1754   protected JList measureList;
1755   protected JList measure2List;
1756   protected JCheckBox setCheck;
1757   protected JCheckBox typeCheck;
1758   protected JCheckBox featureCheck;
1759   protected JProgressBar progressBar;
1760   protected JCheckBox verboseOptionCheckBox;
1761 
1762   // actions
1763   protected OpenDocumentAction openDocumentAction;
1764   protected OpenAnnotationDiffAction openAnnotationDiffAction;
1765   protected ExportToHtmlAction exportToHtmlAction;
1766   protected ReloadCacheAction reloadCacheAction;
1767   protected CompareAction compareAction;
1768 }