/* * POSTagger.java * * Copyright (c) 1995-2012, The University of Sheffield. See the file * COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt * * This file is part of GATE (see http://gate.ac.uk/), and is free * software, licenced under the GNU Library General Public License, * Version 2, June 1991 (in the distribution as file licence.html, * and also available at http://gate.ac.uk/gate/licence.html). * * HepTag was originally written by Mark Hepple, this version contains * modifications by Valentin Tablan and Niraj Aswani. * * $Id: POSTagger.java 17605 2014-03-09 10:15:34Z markagreenwood $ */ /* * INSTRUCTIONS for STAND-ALONE USE * * SYNOPSIS * java hepple.postag.POSTagger [options] file1 [file2 ...] * OPTIONS: * -h, --help : displays this message * -l, --lexicon <lexicon file> : uses specified lexicon * -r, --rules <rules file> : uses specified rules * * NOTE: requires gnu.getopt package */ /** * Title: HepTag * Description: Mark Hepple's POS tagger * Copyright: Copyright (c) 2001 * Company: University of Sheffield * @author Mark Hepple * @version 1.0 */ package hepple.postag; import gate.util.BomStrippingInputStreamReader; import gnu.getopt.Getopt; import gnu.getopt.LongOpt; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.StringTokenizer; import org.apache.commons.io.IOUtils; /** * A Java POS Tagger * * Author: Mark Hepple (hepple@dcs.shef.ac.uk) * * Input: An ascii text file in "Brill input format", i.e. one * sentence per line, tokens separated by spaces. * * Output: Same text with each token tagged, i.e. "token" -> "token/tag". * Output is just streamed to std-output, so commonly will direct * into some target file. * * Revision: 13/9/00. Version 1.0. * * Comments: * * Implements a version of the decision list based tagging method * described in: * * M. Hepple. 2000. Independence and Commitment: Assumptions for Rapid * Training and Execution of Rule-based Part-of-Speech Taggers. * Proceedings of the 38th Annual Meeting of the Association for * Computational Linguistics (ACL-2000). Hong Kong, October 2000. * * Modified by Niraj Aswani/Ian Roberts to allow explicit specification of the * character encoding to use when reading rules and lexicon files. * * $Id: POSTagger.java 17605 2014-03-09 10:15:34Z markagreenwood $ * */ public class POSTagger { // static final int MAXTAGS = 200; protected Map<String, List<Rule>> rules; // public Rule[] rules = new Rule[MAXTAGS]; // public Rule[] lastRules = new Rule[MAXTAGS]; Lexicon lexicon; private String encoding; static final String staart = "STAART"; private String[] staartLex = { staart }; private String[] deflex_NNP = { "NNP"}; private String[] deflex_JJ = { "JJ"}; private String[] deflex_CD = { "CD"}; private String[] deflex_NNS = { "NNS"}; private String[] deflex_RB = { "RB"}; private String[] deflex_VBG = { "VBG"}; private String[] deflex_NN = { "NN"}; public String[] wordBuff = { staart,staart,staart,staart, staart,staart,staart }; public String[] tagBuff = { staart,staart,staart,staart, staart,staart,staart }; public String[][] lexBuff = { staartLex,staartLex,staartLex, staartLex,staartLex,staartLex, staartLex }; /** * Construct a POS tagger using the platform's native encoding to read the * lexicon and rules files. */ public POSTagger(URL lexiconURL, URL rulesURL) throws InvalidRuleException, IOException { this(lexiconURL, rulesURL, null); } /** * Construct a POS tagger using the specified encoding to read the lexicon * and rules files. */ public POSTagger(URL lexiconURL, URL rulesURL, String encoding) throws InvalidRuleException, IOException{ this.encoding = encoding; this.lexicon = new Lexicon(lexiconURL, encoding); rules = new HashMap<String, List<Rule>>(); readRules(rulesURL); } /** * Creates a new rule of the required type according to the provided ID. * @param ruleId the ID for the rule to be created */ public Rule createNewRule(String ruleId) throws InvalidRuleException{ try{ String className = "hepple.postag.rules.Rule_" + ruleId; Class<?> ruleClass = Class.forName(className); return (Rule)ruleClass.newInstance(); }catch(Exception e){ throw new InvalidRuleException("Could not create rule " + ruleId + "!\n" + e.toString()); } } /** * Runs the tagger over a set of sentences. * @param sentences a {@link java.util.List} of {@link java.util.List}s * of words to be tagged. Each list is a sentence represented as a list of * words. * @return a {@link java.util.List} of {@link java.util.List}s of * {@link java.lang.String}[]. A list of tagged sentences, each sentence * being itself a list having pairs of strings as elements with * the word on the first position and the tag on the second. */ public List<List<String[]>> runTagger(List<List<String>> sentences){ List<List<String[]>> output = new ArrayList<List<String[]>>(); List<String[]> taggedSentence = new ArrayList<String[]>(); Iterator<List<String>> sentencesIter = sentences.iterator(); while(sentencesIter.hasNext()){ List<String> sentence = sentencesIter.next(); Iterator<String> wordsIter = sentence.iterator(); while(wordsIter.hasNext()){ String newWord = wordsIter.next(); oneStep(newWord, taggedSentence); }//while(wordsIter.hasNext()) //finished adding all the words from a sentence, add six more //staarts to flush all words out of the tagging buffer for(int i = 0; i < 6; i++){ oneStep(staart, taggedSentence); } //we have a new finished sentence output.add(taggedSentence); taggedSentence = new ArrayList<String[]>(); }//while(sentencesIter.hasNext()) return output; } /** * Adds a new word to the window of 7 words (on the last position) and tags * the word currently in the middle (i.e. on position 3). This function * also reads the word on the first position and adds its tag to the * taggedSentence structure as this word would be lost at the next advance. * If this word completes a sentence then it returns true otherwise it * returns false. * @param word the new word * @param taggedSentence a List of pairs of strings representing the results * of tagging the current sentence so far. * @return returns true if a full sentence is now tagged, otherwise false. */ protected boolean oneStep(String word, List<String[]> taggedSentence){ //add the new word at the end of the text window for (int i=1 ; i<7 ; i++) { wordBuff[i-1] = wordBuff[i]; tagBuff[i-1] = tagBuff[i]; lexBuff[i-1] = lexBuff[i]; } wordBuff[6] = word; lexBuff[6] = classifyWord(word); tagBuff[6] = lexBuff[6][0]; //apply the rules to the word in the middle of the text window //Try to fire a rule for the current lexical entry. It may be the case that //no rule applies. List<Rule> rulesToApply = rules.get(lexBuff[3][0]); if(rulesToApply != null && rulesToApply.size() > 0){ Iterator<Rule> rulesIter = rulesToApply.iterator(); //find the first rule that applies, fire it and stop. while(rulesIter.hasNext() && !(rulesIter.next()).apply(this)){} } //save the tagged word from the first position String taggedWord = wordBuff[0]; if(taggedWord != staart){ taggedSentence.add(new String[]{taggedWord, tagBuff[0]}); if(wordBuff[1] == staart){ //wordTag[0] was the end of a sentence return true; }//if(wordBuff[1] == staart) }//if(taggedWord != staart) return false; }//protected List oneStep(String word, List taggedSentence) /** * Reads the rules from the rules input file */ @SuppressWarnings("resource") public void readRules(URL rulesURL) throws IOException, InvalidRuleException{ BufferedReader rulesReader = null; try { if(encoding == null) { rulesReader = new BomStrippingInputStreamReader(rulesURL.openStream()); } else { rulesReader = new BomStrippingInputStreamReader(rulesURL.openStream(), this.encoding); } String line; Rule newRule; line = rulesReader.readLine(); while(line != null){ List<String> ruleParts = new ArrayList<String>(); StringTokenizer tokens = new StringTokenizer(line); while (tokens.hasMoreTokens()) ruleParts.add(tokens.nextToken()); if (ruleParts.size() < 3) throw new InvalidRuleException(line); newRule = createNewRule(ruleParts.get(2)); newRule.initialise(ruleParts); List<Rule> existingRules = rules.get(newRule.from); if(existingRules == null){ existingRules = new ArrayList<Rule>(); rules.put(newRule.from, existingRules); } existingRules.add(newRule); line = rulesReader.readLine(); }//while(line != null) } finally { IOUtils.closeQuietly(rulesReader); } }//public void readRules() public void showRules(){ System.out.println(rules); } /** * Attempts to classify an unknown word. * @param wd the word to be classified */ protected String[] classifyWord(String wd){ String[] result; if (staart.equals(wd)) return staartLex; List<String> categories = lexicon.get(wd); if(categories != null){ result = new String[categories.size()]; for(int i = 0; i < result.length; i++){ result[i] = categories.get(i); } return result; } //no lexical entry for the word. Try to guess if ('A' <= wd.charAt(0) && wd.charAt(0) <= 'Z') return deflex_NNP; for (int i=1 ; i < wd.length()-1 ; i++) if (wd.charAt(i) == '-') return deflex_JJ; for (int i=0 ; i < wd.length() ; i++) if ('0' <= wd.charAt(i) && wd.charAt(i) <= '9') return deflex_CD; if (wd.endsWith("ed") || wd.endsWith("us") || wd.endsWith("ic") || wd.endsWith("ble") || wd.endsWith("ive") || wd.endsWith("ary") || wd.endsWith("ful") || wd.endsWith("ical") || wd.endsWith("less")) return deflex_JJ; if (wd.endsWith("s")) return deflex_NNS; if (wd.endsWith("ly")) return deflex_RB; if (wd.endsWith("ing")) return deflex_VBG; return deflex_NN; }//private String[] classifyWord(String wd) /** * Main method. Runs the tagger using the arguments to find the resources * to be used for initialisation and the input file. */ public static void main(String[] args){ if(args.length == 0) help(); try{ LongOpt[] options = new LongOpt[]{ new LongOpt("help", LongOpt.NO_ARGUMENT, null, 'h'), new LongOpt("lexicon", LongOpt.NO_ARGUMENT, null, 'l'), new LongOpt("rules", LongOpt.NO_ARGUMENT, null, 'r') }; Getopt getopt = new Getopt("HepTag", args, "hl:r:", options); String lexiconUrlString = null; String rulesUrlString = null; int opt; while( (opt = getopt.getopt()) != -1 ){ switch(opt) { // -h case 'h':{ help(); System.exit(0); break; } // -l new lexicon case 'l':{ lexiconUrlString = getopt.getOptarg(); break; } // -l new lexicon case 'r':{ rulesUrlString = getopt.getOptarg(); break; } default:{ System.err.println("Invalid option " + args[getopt.getOptind() -1] + "!"); System.exit(1); } }//switch(opt) }//while( (opt = g.getopt()) != -1 ) String[] fileNames = new String[args.length - getopt.getOptind()]; for(int i = getopt.getOptind(); i < args.length; i++){ fileNames[i - getopt.getOptind()] = args[i]; } URL lexiconURL = (lexiconUrlString == null) ? POSTagger.class. getResource("/hepple/resources/sample_lexicon") : new File(lexiconUrlString).toURI().toURL(); URL rulesURL = (rulesUrlString == null) ? POSTagger.class. getResource("/hepple/resources/sample_ruleset.big") : new File(rulesUrlString).toURI().toURL(); POSTagger tagger = new POSTagger(lexiconURL, rulesURL); for(int i = 0; i < fileNames.length; i++){ String file = fileNames[i]; BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(file)); String line = reader.readLine(); while(line != null){ StringTokenizer tokens = new StringTokenizer(line); List<String> sentence = new ArrayList<String>(); while(tokens.hasMoreTokens()) sentence.add(tokens.nextToken()); List<List<String>> sentences = new ArrayList<List<String>>(); sentences.add(sentence); List<List<String[]>> result = tagger.runTagger(sentences); Iterator<List<String[]>> iter = result.iterator(); while(iter.hasNext()){ List<String[]> sentenceFromTagger = iter.next(); Iterator<String[]> sentIter = sentenceFromTagger.iterator(); while(sentIter.hasNext()){ String[] tag = sentIter.next(); System.out.print(tag[0] + "/" + tag[1]); if(sentIter.hasNext()) System.out.print(" "); else System.out.println(); }//while(sentIter.hasNext()) }//while(iter.hasNext()) line = reader.readLine(); }//while(line != null) } finally { IOUtils.closeQuietly(reader); } // // // // List result = tagger.runTagger(readInput(file)); // Iterator iter = result.iterator(); // while(iter.hasNext()){ // List sentence = (List)iter.next(); // Iterator sentIter = sentence.iterator(); // while(sentIter.hasNext()){ // String[] tag = (String[])sentIter.next(); // System.out.print(tag[0] + "/" + tag[1]); // if(sentIter.hasNext()) System.out.print(" "); // else System.out.println(); // }//while(sentIter.hasNext()) // }//while(iter.hasNext()) }//for(int i = 0; i < fileNames.length; i++) }catch(Exception e){ e.printStackTrace(); } }//public static void main(String[] args) /** * Prints the help message */ private static void help(){ System.out.println( "NAME\n" + "HepTag - a Part-of-Speech tagger\n" + "see http://www.dcs.shef.ac.uk/~hepple/papers/acl00/abstract.html \n\n" + "SYNOPSIS\n\tjava hepple.postag.POSTagger [options] file1 [file2 ...]\n\n" + "OPTIONS:\n" + "-h, --help \n\tdisplays this message\n" + "-l, --lexicon <lexicon file>\n\tuses specified lexicon\n" + "-r, --rules <rules file>\n\tuses specified rules"); } /** * Reads one input file and creates the structure needed by the tagger * for input. */ @SuppressWarnings("unused") private static List<List<String>> readInput(String file) throws IOException{ BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(file)); String line = reader.readLine(); List<List<String>> result = new ArrayList<List<String>>(); while(line != null){ StringTokenizer tokens = new StringTokenizer(line); List<String> sentence = new ArrayList<String>(); while(tokens.hasMoreTokens()) sentence.add(tokens.nextToken()); result.add(sentence); line = reader.readLine(); }//while(line != null) return result; } finally { IOUtils.closeQuietly(reader); } }//private static List readInput(File file) throws IOException }//public class POSTagger