%% LaTeX Beamer presentation template (requires beamer package)
%% see http://latex-beamer.sourceforge.net/
%% idea contributed by H. Turgut Uyar
%% template based on a template by Till Tantau
%% this template is still evolving - it might differ in future releases!


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%  Uncomment version below for creating normal slides  %%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass{beamer}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%  Uncomment version below for creating handouts  %%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\documentclass[handout]{beamer}
%\usepackage{pgfpages}
%\pgfpagesuselayout{4 on 1}[a4paper, landscape, border shrink=5mm]

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\mode<presentation>

\newif\ifsolutions

% Swap this to solutionstrue to include solution slides for the hands-on
% exercises
\solutionstrue
%\solutionsfalse

% Set up an \omitfromhandout command to specify slides that should be omitted
% when generating the handout.
\makeatletter
\newwrite\tf@ofh
\immediate\openout\tf@ofh\jobname.ofh
\makeatother

\newcommand{\omitfromhandout}{\addtocontents{ofh}{\thepage}}

\renewcommand<>{\omitfromhandout}[0]{{\only#1{\beameroriginal{\omitfromhandout}}}}

\usetheme{GATE}
\input{slides-preamble}

\title{GATE APIs}
\subtitle{Track II, Module 6}


% This is only inserted into the PDF information catalog. Can be left
% out.
\subject{GATE Embedded API}

% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

%\pgfdeclareimage[height=2cm]{university-logo}{../../images/logo.png}
%\logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
\AtBeginSubsection[]
{
\begin{frame}<beamer>
\frametitle{Outline}
\tableofcontents[currentsection, currentsubsection]
\end{frame}
}

\begin{document}

%\addtocounter{framenumber}{-1}\framenumbersfalse
%\begin{frame}
%\frametitle{Certification and other announcements}
%\begin{itemize}
%\item Please let us know which modules you intend to take
%  \begin{itemize}
%  \item You will have time for up to 4 modules, you need 3 passes to get a
%    certificate
%  \item You can choose modules from other tracks, e.g. if you already know GATE
%    Developer you could take module 1
%  \end{itemize}
%\item Exams are based on the presentations -- everything you need to know to
%  pass is in the handouts.
%\item Q\&A session 2pm today if you have any queries/concerns
%\item Also note: today's afternoon talk is in Middleton, not Sharman
%\end{itemize}
%\end{frame}
%\framenumberstrue

\begin{frame}
\titlepage
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Outline}
\tableofcontents
% You might wish to add the option [pausesections]
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Using Java in JAPE}

\subsection{Basic JAPE}

\begin{frame}
\frametitle{JAPE}
\framesubtitle{Pattern matching over annotations}
\begin{itemize}
\item JAPE is a language for doing regular-expression-style pattern matching
  over \emph{annotations} rather than text.
\item Each JAPE rule consists of
  \begin{itemize}
  \item Left hand side specifying the patterns to match
  \item Right hand side specifying what to do when a match is found
  \end{itemize}
\item JAPE rules combine to create a phase
\item Phases combine to create a grammar
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{An Example JAPE Rule}
\begin{lstlisting}
Rule: University1 
(
  {Token.string == "University"} 
  {Token.string == "of"}
  {Lookup.minorType == city} 
):orgName 
-->
:orgName.Organisation = 
  {kind = "university", rule = "University1"}
\end{lstlisting}
Left hand side specifies annotations to match, optionally labelling
some of them for use on the right hand side.
\end{frame}

\begin{frame}[fragile]
\frametitle{LHS Patterns}
\framesubtitle{Elements}
Left hand side of the rule specifies the pattern to match, in various ways
\begin{itemize}
\item Annotation type: \lstinline!{Token}!
\item Feature constraints: 
  \begin{itemize}
  \item \lstinline!{Token.string == "University"}!
  \item \lstinline!{Token.length > 4}!
  \item Also supports \verb|<|, \verb|<=|, \verb|>=|, \verb|!=| and regular
    expressions \verb|=~|, \verb|==~|, \verb|!~|, \verb|!=~|.
  \end{itemize}
\item Negative constraints: \lstinline|{Token.length > 4, !Lookup.majorType == "stopword"}|
  \begin{itemize}
  \item This matches a Token of more than 4 characters that does not start at
    the same location as a "stopword" Lookup.
  \end{itemize}
\item Overlap constraints:
  \lstinline|{Person within {Section.title == "authors"}}|
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{LHS Patterns}
\framesubtitle{Combinations}
Pattern elements can be combined in various ways
\begin{itemize}
\item Sequencing: \lstinline|{Token}{Token}|
\item Alternatives: \lstinline!{Token} | {Lookup}!
\item Grouping with parentheses
\end{itemize}
Usual regular expression multiplicity operators
\begin{itemize}
\item zero-or-one: \lstinline!({MyAnnot})?!
\item zero-or-more: \lstinline!({MyAnnot})*!
\item one-or-more: \lstinline!({MyAnnot})+!
\item exactly $n$: \lstinline!({MyAnnot})[n]!
\item between $n$ and $m$ (inclusive): \lstinline!({MyAnnot})[n,m]!
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{LHS Patterns}
\framesubtitle{Labelling}
Groups can be labelled.  This has no effect on the matching process, but makes
matched annotations available to the RHS
\begin{lstlisting}
(
  {Token.string == "University"} 
  {Token.string == "of"}
  ({Lookup.minorType == city}):uniTown
):orgName 
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{RHS Actions}
On the RHS, you can use the labels from the LHS to create new annotations:
\begin{lstlisting}[firstnumber=6]
-->
:uniTown.UniversityTown = {},
:orgName.Organisation =
  {kind = "university", rule = "University1"}
\end{lstlisting}
The \verb|:label.AnnotationType = {features}| syntax creates a new
annotation of the given type whose span covers all the annotations bound to the
label.
\begin{itemize}
\item so the \verb|Organisation| annotation will span from the start of the
``University'' Token to the end of the Lookup.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Macros}
\begin{itemize}
\item You may find yourself re-using the same patterns in several places in a
  grammar.
  \begin{itemize}
  \item e.g. \lstinline!({Token.string ==~ "[A-Z]"}({Token.string == "."})?)+!
    to match initials.
  \end{itemize}
\item JAPE allows you to define \emph{macros} - labelled patterns that can be
  re-used.
\end{itemize}
\begin{lstlisting}[texcl]
Macro: INITIALS
({Token.string ==~ "[A-Z]"}({Token.string == "."})?)+

Rule: InitialsAndSurname
( (INITIALS)?
  {Token.orth == "upperInitial"} ):per
-->
:per.Person = {rule = "InitialsAndSurname"}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Templates}
\begin{itemize}
\item Templates are to values as macros are to pattern fragments.
\item Declare a template once, reference it many times.
\item Template value can be a quoted string, number or boolean
  (\lstinline!true! or \lstinline!false!).
\item Template reference can go anywhere a quoted string could go.
\end{itemize}
\begin{lstlisting}[texcl]
Template: threshold = 0.6
Template: source = "Interesting location finder"

Rule: IsInteresting
({Location.score > [threshold]}):loc
-->
:loc.Entity = { kind = "Location", source = [source]}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Templates (cont)}
\begin{itemize}
\item String templates can have \emph{parameters}, parameter values supplied in
  the call.
\item Useful if you have many similar strings in your grammar.
\end{itemize}
\begin{lstlisting}[texcl]
Template:
  wp = "http://${lang}.wikipedia.org/wiki/${page}"

Rule: EnglishWPCat
({a.href =~ [wp lang="en", page="Category:"]}):wp
-->
:wp.WPCategory = { lang = "en" }
\end{lstlisting}
\begin{itemize}
\item In a multi-phase grammar, templates and macros declared in one phase can
  be used in later phases.
\end{itemize}
\end{frame}

\subsection{Java on the RHS}

\begin{frame}
\frametitle{Beyond Simple Actions}
It's often useful to do more complex operations on the RHS than simply adding
annotations, e.g.
\begin{itemize}
\item Set a new feature on one of the matched annotations
\item Delete annotations from the input
\item More complex feature value mappings, e.g. concatenate several LHS
  features to make one RHS one.
\item Collect statistics, e.g. count the number of matched annotations and
  store the count as a document feature.
\item Populate an ontology (later).
\end{itemize}
JAPE has no special syntax for these operations, but allows blocks of arbitrary
Java code on the RHS.
\end{frame}

\begin{frame}[fragile]
\frametitle{Java on the RHS}
\begin{lstlisting}
Rule: HelloWorld
(
  {Token.string == "Hello"}
  {Token.string == "World"}
):hello
-->
{
  System.out.println("Hello world");
}
\end{lstlisting}
The RHS of a JAPE rule can have any number of \verb|:bind.Type = {}| assignment
expressions and blocks of Java code, separated by commas.
\end{frame}

\begin{frame}[fragile]
\frametitle{How JAPE Rules are Compiled}
For each JAPE rule, GATE creates a Java class
\begin{lstlisting}
package japeactionclasses;
// various imports, see below

public class /* generated class name */
    implements RhsAction {
  public void doit(
      Document doc,
      Map<String, AnnotationSet> bindings,
      AnnotationSet annotations, // deprecated
      AnnotationSet inputAS,
      AnnotationSet outputAS,
      Ontology ontology) throws JapeException {
    // ...
  }
}
\end{lstlisting}
\end{frame}

\begin{frame}
\frametitle{JAPE Action Classes}
\begin{itemize}
\item Each block or assignment on the RHS becomes a block of Java code.
\item These blocks are concatenated together to make the body of the
\texttt{doit} method.
  \begin{itemize}
  \item Local variables are local to each block, not shared.
  \end{itemize}
\item At runtime, whenever the rule matches, \texttt{doit} is called.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Java Block Parameters}
The parameters available to Java RHS blocks are:
\begin{description}
\item[doc] The document currently being processed.
\item[inputAS] The \texttt{AnnotationSet} specified by the
  \texttt{inputASName} runtime parameter to the JAPE transducer PR. Read or
  delete annotations from here.
\item[outputAS] The \texttt{AnnotationSet} specified by the
  \texttt{outputASName} runtime parameter to the JAPE transducer PR.  Create new
  annotations in here.
\item[ontology] The ontology (if any) provided as a runtime parameter to the
  JAPE transducer PR.
\item[bindings] The bindings map\ldots
\end{description}
\end{frame}


\begin{frame}[fragile]
\frametitle{Bindings}
\begin{itemize}
\item \texttt{bindings} is a \texttt{Map} from string to \texttt{AnnotationSet}
\item Keys are labels from the LHS.
\item Values are the annotations matched by the label.
\end{itemize}
\begin{lstlisting}
(
  {Token.string == "University"} 
  {Token.string == "of"}
  ({Lookup.minorType == city}):uniTown
):orgName 
\end{lstlisting}
\begin{itemize}
\item \lstinline!bindings.get("uniTown")! contains one annotation (the
  \texttt{Lookup})
\item \lstinline!bindings.get("orgName")! contains three annotations (two
  \texttt{Token}s plus the \texttt{Lookup})
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Hands-on exercises}
\begin{itemize}
\item The easiest way to experiment with JAPE is to use GATE Developer.
\item The \verb|hands-on| directory contains a number of sample JAPE files for
  you to modify, which will be described for each individual exercise.
\item There is an \verb|.xgapp| file for each exercise to load the right PRs
  and documents.
  \begin{itemize}
  \item Good idea to \emph{disable} session saving using Options $\rightarrow$
    Configuration $\rightarrow$ Advanced (or GATE 6.0-beta1 $\rightarrow$
    Preferences $\rightarrow$ Advanced on Mac OS X).
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 1: A simple JAPE RHS}
\begin{itemize}
\item Start GATE Developer.
\item Load \verb|hands-on/jape/exercise1.xgapp|
\item This is the default ANNIE application with an additional JAPE transducer
  ``exercise 1'' at the end.
\item This transducer loads the file \verb|hands-on/jape/resources/simple.jape|,
  which contains a single simple JAPE rule.
\item Modify the Java RHS block to print out the type and features of each
  annotation the rule matches.  You need to right click the ``Exercise 1
  Transducer'' and reinitialize after saving the \verb|.jape| file.
\item Test it by running the ``Exercise 1'' application.
\end{itemize}
\end{frame}

\ifsolutions
\addtocounter{framenumber}{-1}\framenumbersfalse
\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 1: Solution}
A possible solution:
\begin{lstlisting}
Rule: ListEntities
({Person}|{Organization}|{Location}):ent
-->
{
  AnnotationSet ents = bindings.get("ent");
  for(Annotation e : ents) {
    System.out.println("Found " + e.getType()
        + " annotation");
    System.out.println("  features: "
        + e.getFeatures());
  }
}
\end{lstlisting}
\end{frame}
\framenumberstrue
\fi

\begin{frame}[fragile]
\frametitle{Imports}
\begin{itemize}
\item By default, every action class imports java.io.*, java.util.*, gate.*,
  gate.jape.*, gate.creole.ontology.*, gate.annotation.*, and gate.util.*.
\item So classes from these packages can be used unqualified in RHS blocks.
\item You can add additional imports by putting an import block at the top of
  the JAPE file, before the \texttt{Phase:} line:
\end{itemize}
\begin{lstlisting}
Imports: {
  import my.pkg.*;
  import static gate.Utils.*;
}
\end{lstlisting}
You can import any class available in the GATE core or in any loaded plugin.
\end{frame}

\begin{frame}[fragile]
\frametitle{Named Java Blocks}
\begin{lstlisting}
-->
:uniTown{
  uniTownAnnots.iterator().next().getFeatures()
    .put("hasUniversity", Boolean.TRUE);
}
\end{lstlisting}
\begin{itemize}
\item You can label a Java block with a label from the LHS
\item The block will only be called if there is at least one annotation bound
  to the label
\item Within the Java block there is a variable \texttt{\emph{label}Annots}
  referring to the \lstinline!AnnotationSet! bound to the label
  \begin{itemize}
  \item i.e. \lstinline!AnnotationSet xyAnnots = bindings.get("xy")!
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exceptions}
\begin{itemize}
\item Any \lstinline!JapeException! or \lstinline!RuntimeException! thrown by a
  Java RHS block will cause the JAPE Transducer PR to fail with an
  \lstinline!ExecutionException!
\item For non-fatal errors in a RHS block you can throw a
  \lstinline!gate.jape.NonFatalJapeException!
\item This will print debugging information (phase name, rule name, file and
  line number) but will not abort the transducer execution.
  \begin{itemize}
  \item However it will interrupt this rule, i.e. if there is more than one
    block or assignment on the RHS, the ones after the \lstinline!throw! will
    not run.
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Returning from RHS blocks}
\begin{itemize}
\item You can \lstinline!return! from a Java RHS block, which prevents any
  later blocks or assignments for that rule from running, e.g.
\end{itemize}
\begin{lstlisting}[texcl]
-->
:uniTown{
  String townString = doc.getContent().getContent(
        uniTownAnnots.firstNode().getOffset(),
        uniTownAnnots.lastNode().getOffset())
      .toString();
  // don't add an annotation if this town has been seen before.  If we
  // return, the UniversityTown annotation will not be created.
  if(!((Set)doc.getFeatures().get("knownTowns"))
      .add(townString)) return;
},
:uniTown.UniversityTown = {}
\end{lstlisting}
\end{frame}

\subsection{Common idioms}

\begin{frame}[fragile]
\frametitle{Common Idioms for Java RHS}
Setting a new feature on one of the matched annotations
\begin{lstlisting}[texcl]
Rule: LcString
({Token}):tok
-->
:tok {
  for(Annotation a : tokAnnots) {
    // get the FeatureMap for the annotation
    FeatureMap fm = a.getFeatures();
    // get the ``string'' feature
    String str = (String)fm.get("string");
    // convert it to lower case and store
    fm.put("lcString", str.toLowerCase());
  }
}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 2: Modifying Existing Annotations}
\begin{itemize}
\item Load \verb|hands-on/jape/exercise2.xgapp|
\item As before, this is ANNIE plus an extra transducer, this time loading
  \verb|hands-on/jape/resources/general-pos.jape|.
\item Modify the Java RHS block to add a \verb|generalCategory| feature to the
  matched \verb|Token| annotation holding the first two characters of the POS
  tag (the \verb|category| feature).
\item Remember to reinitialize the ``Exercise 2 Transducer'' after editing the
  JAPE file.
\item Test it by running the ``Exercise 2'' application.
\end{itemize}
\end{frame}

\ifsolutions
\addtocounter{framenumber}{-1}\framenumbersfalse
\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 2: Solution}
A possible solution:
\begin{lstlisting}
Rule: GeneralizePOSTag
({Token}):tok
-->
:tok {
  for(Annotation t : tokAnnots) {
    String pos = (String)t.getFeatures()
                          .get("category");
    if(pos != null) {
      int gpLen = pos.length();
      if(gpLen > 2) gpLen = 2;
      t.getFeatures().put("generalCategory",
          pos.substring(0, gpLen));
    }
  }
}
\end{lstlisting}
\end{frame}
\framenumberstrue
\fi

\begin{frame}[fragile]
\frametitle{Common Idioms for Java RHS}
Removing matched annotations from the input
\begin{lstlisting}
Rule: Location
({Lookup.majorType = "location"}):loc
-->
:loc.Location = { kind = :loc.Location.minorType,
      rule = "Location"},
:loc {
  inputAS.removeAll(locAnnots);
}
\end{lstlisting}
This can be useful to stop later phases matching the same annotations again.
\end{frame}

\begin{frame}[fragile]
\frametitle{Common Idioms for Java RHS}
Accessing the string covered by a match
\begin{lstlisting}[texcl]
Rule: Location
({Lookup.majorType = "location"}):loc
-->
:loc {
  try {
    String str = doc.getContent().getContent(
        locAnnots.firstNode().getOffset(),
        locAnnots.lastNode().getOffset())
      .toString();
  }
  catch(InvalidOffsetException e) {
    // can't happen, but won't compile without the catch
  }
}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Utility methods}
\begin{itemize}
\item \lstinline!gate.Utils! provides static utility methods to make common
  tasks easier
  \begin{itemize}
  \item {\scriptsize \verb|http://gate.ac.uk/gate/doc/javadoc/gate/Utils.html|}
  \end{itemize}
\item Add an \lstinline!import static gate.Utils.*;! to your
\lstinline!Imports:! block to use them.
\item Accessing the string becomes \lstinline!stringFor(doc, locAnnots)!
\item This is also useful for division of labour
  \begin{itemize}
  \item Java programmer writes utility class
  \item JAPE expert writes rules, importing utility methods
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Example: start and end}
To get the start and end offsets of an \lstinline!Annotation!,
\lstinline!AnnotationSet! or \lstinline!Document!.
\begin{lstlisting}[emph={start,end},emphstyle=\alert]
Rule: NPTokens
({NounPhrase}):np
-->
:np {
  List<String> posTags = new ArrayList<String>();
  for(Annotation tok : inputAS.get("Token")
      .getContained(start(npAnnots), end(npAnnots))) {
    posTags.add(
        (String)tok.getFeatures().get("category"));
  }
  FeatureMap fm =
    npAnnots.iterator().next().getFeatures();
  fm.put("posTags", posTags);
  fm.put("numTokens", (long)posTags.size());
}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 3: Working with Contained Annotations}
\begin{itemize}
\item Load \verb|hands-on/jape/exercise3.xgapp|
\item As before, this is ANNIE plus an extra transducer, this time loading
  \verb|hands-on/jape/resources/exercise3-main.jape|.
\item This is a multiphase grammar containing the \verb|general-pos.jape| from
  exercise 2 plus \verb|num-nouns.jape|.
\item Modify the Java RHS block in \verb|num-nouns.jape| to count the number of
  nouns in the matched \verb|Sentence| and add this count as a feature on the
  sentence annotation.
\item Remember to reinitialize the ``Exercise 3 Transducer'' after editing the
  JAPE file.
\item Test it by running the ``Exercise 3'' application.
\end{itemize}
\end{frame}

\ifsolutions
\addtocounter{framenumber}{-2}\framenumbersfalse
\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 3: Solution}
A possible solution:
\begin{lstlisting}
Imports: { import static gate.Utils.*; }
Phase: NumNouns
Input: Sentence
Options: control = appelt

Rule: CountNouns
({Sentence}):sent
-->
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 3: Solution (continued)}
\begin{lstlisting}[firstnumber=last]
:sent {
  AnnotationSet tokens = inputAS.get("Token")
    .getContained(start(sentAnnots), end(sentAnnots));
  long numNouns = 0;
  for(Annotation t : tokens) {
    if("NN".equals(t.getFeatures()
                    .get("generalCategory"))) {
      numNouns++;
    }
  }
  sentAnnots.iterator().next().getFeatures()
    .put("numNouns", numNouns);
}
\end{lstlisting}
\end{frame}
\framenumberstrue
\fi

\begin{frame}[fragile]
\frametitle{Passing state between rules}
To pass state between rules, use document features:
\begin{lstlisting}
Rule: Section
({SectionHeading}):sect
-->
:sect {
  doc.getFeatures().put("currentSection",
      stringFor(doc, sectAnnots));
}

Rule: Entity
({Entity}):ent
-->
:ent {
  entAnnots.iterator().next().getFeatures()
    .put("inSection",
         doc.getFeatures().get("currentSection"));
}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Passing state between rules}
\begin{itemize}
\item Remember from yesterday - a \lstinline!FeatureMap! can hold any Java
  object.
\item So can pass complex structures between rules, not limited to simple
  strings.
\end{itemize}
\end{frame}

%TODO come up with some hands on exercises

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{The GATE Ontology API}

\subsection{5 minute guide to ontologies}

\begin{frame}
\frametitle{Ontologies}
\framesubtitle{A 5 minute introduction}
\begin{columns}
  \begin{column}{0.7\textwidth}
    \begin{itemize}
    \item A set of concepts and relationships between them.
    \item GATE uses the \emph{OWL} formalism for ontologies
    \item Classes, subclasses, instances, relationships
    \item Multiple inheritance
      \begin{itemize}
      \item a class can have many superclasses
      \item an instance can belong to many classes
      \end{itemize}
    \end{itemize}
  \end{column}
  \begin{column}{0.3\textwidth}
    \includegraphics[width=0.9\textwidth]{protonu-company-class}
  \end{column}
\end{columns}
\end{frame}

\begin{frame}
\frametitle{Why Ontologies?}
\begin{itemize}
\item Semantic annotation: rather than just annotating the word ``Sheffield''
  as a location, link it to an ontology instance
  \begin{itemize}
  \item Sheffield, UK rather than Sheffield, Massachusetts or Sheffield,
    Tasmania, etc.
  \end{itemize}
\item Reasoning
  \begin{itemize}
  \item Ontology tells us that this particular Sheffield is part of the country
    called the United Kingdom, which is part of the continent Europe.
  \item So we can infer that this document mentions a city in Europe.
  \end{itemize}
\item Relation extraction: match patterns in text and use them to add new
  information to the ontology.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Ontologies}
\framesubtitle{Properties}
\begin{itemize}
\item \emph{Properties} represent relationships between, and data about,
  instances.
\item Properties can have hierarchy.
\end{itemize}
\includegraphics[width=0.9\textwidth]{protonu-has-relative-prop}
\begin{itemize}
\item \emph{Object} properties relate one instance to another (DCS
  \emph{partOf} University of Sheffield) --- domain and range specify
  which classes the instances must belong to
\item Can be symmetric, transitive
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Ontologies}
\framesubtitle{Datatype Properties}
\includegraphics[width=0.9\textwidth]{protonu-description-prop}
\begin{itemize}
\item \emph{Datatype} properties attach simple data (\emph{literals}) to
  instances.
\item Available data types are taken from XML Schema.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Ontologies}
\framesubtitle{Annotation Properties}
\includegraphics[width=0.9\textwidth]{protonu-month-annotation-prop}
\begin{itemize}
\item \emph{Annotation} properties used to annotate classes, instances and
  other properties (collectively known as \emph{resources}, confusingly).
\item Similar to datatype properties, but those can only be attached to
  instances, not classes.
\item e.g. RDFS defines properties like \emph{comment} and \emph{label} (a
  human-readable name for an ontology resource, as opposed to formal name of
  the resource which is a URI).
\end{itemize}
\end{frame}

\subsection{Ontologies in GATE Embedded}

\begin{frame}[fragile]
\frametitle{Ontologies in GATE Embedded}
\begin{itemize}
\item GATE represents ontologies using abstract data model defined by
  interfaces in \lstinline!gate.creole.ontology! package in \verb!gate.jar!
\item \verb!Ontology! interface represents an ontology, \verb!OClass!,
  \verb!OInstance!, \verb!OURI! etc. represent ontology components.
\item Implementation provided by \verb!Ontology! plugin, based on OWLIM version
  3.
  \begin{itemize}
  \item Alternative OWLIM 2-based implementation in \verb!Ontology_OWLIM2!
    plugin for backwards compatibility only
  \item Not possible to load both plugins at the same time.
  \end{itemize}
\item You need to load the plugin in order to create an \verb!Ontology!
  object, but code should only interact with the interfaces.
  \begin{itemize}
  \item \url{http://gate.ac.uk/gate/doc/javadoc/?gate/creole/ontology/package-summary.html}
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Creating an empty ontology}
\begin{lstlisting}[texcl]
Gate.init();
// load the Ontology plugin
Gate.getCreoleRegister().registerDirectories(
    new File(Gate.getPluginsHome(), "Ontology")
      .toURI().toURL());

Ontology emptyOnto = (Ontology)Factory.createResource(
    "gate.creole.ontology.impl.sesame.OWLIMOntology");
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Loading an existing OWL file}
More useful is to load an existing ontology.  \verb|OWLIMOntology| can load
RDF-XML, N3, ntriples or turtle format.
\begin{lstlisting}[texcl]
// init GATE and load plugin as before...

URL owl = new File("ontology.owl").toURI().toURL();
FeatureMap params = Factory.newFeatureMap();
params.put("rdfXmlURL", owl);

Ontology theOntology = (Ontology)Factory.createResource(
    "gate.creole.ontology.impl.sesame.OWLIMOntology",
    params);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Under the Covers: Sesame}
\begin{itemize}
\item The Ontology plugin implementation is built on OpenRDF Sesame version 2.
\item \verb|OWLIMOntology| LR creates a Sesame repository using a particular
  configuration of OWLIM as the underlying SAIL (Storage And Inference Layer)
\item Other configurations or SAIL implementations can be used via alternative
  LRs: \verb|CreateSesameOntology| (to create a new repository) and
  \verb|ConnectSesameOntology| (to open an existing one).
  \begin{itemize}
  \item though some parts of the GATE ontology API depend on the reasoning
    provided by OWLIM, so other SAILs may not behave exactly the same.
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Persistent Repositories}
\begin{itemize}
\item When loading an \verb|OWLIMOntology| LR from RDF/ntriples, etc. OWLIM
  parses the source file and builds internal representation
\item Can set \verb|persistent| parameter to \verb|true| and specify a
  \verb|dataDirectoryURL| to store this internal representation on disk as a
  Sesame repository.
\item \verb|ConnectSesameOntology| PR can use the existing repository --- much
  faster to init, particularly for large ontologies (e.g. ~12k instances, 10
  seconds to load from RDF, $<$ 0.2s to open repository).
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exploring the ontology}
\begin{lstlisting}[texcl]
// get all the `top' classes
Set<OClass> tops = ontology.getOClasses(true);

// list them along with their labels
for(OClass c : tops) {
  System.out.println(c.getONodeID() +
      " (" + c.getLabels() + ")");
}

// find a class by URI
OURI uri = ontology.createOURIForName("Person");
OClass personClass = ontology.getOClass(uri);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exploring the ontology}
\begin{lstlisting}[texcl]
// get direct instances of a class
Set<OInstance> people = ontology.getOInstances(
    personClass, OConstants.Closure.DIRECT_CLOSURE);

// get instances of a class or any of its subclasses
Set<OInstance> allPeople = ontology.getOInstances(
    personClass, OConstants.Closure.TRANSITIVE_CLOSURE);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exploring the ontology}
\begin{lstlisting}[texcl]
// get a datatype property
OURI namePropURI = ontology.createOURI(
    "http://example.org/stuff/1.0/hasName");
DatatypeProperty nameProp = ontology
    .getDatatypeProperty(namePropURI);

// find property values for an instance
for(OInstance person : allPeople) {
  List<Literal> names =
      ontology.getDatatypePropertyValues(nameProp);
  for(Literal name : names) {
    System.out.println("Person " + person.getONodeID()
        + " hasName " + name.toTurtle());
  }
}
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exploring the ontology}
\begin{lstlisting}[texcl]
// University of Sheffield instance
OURI uosURI = ontology.createOURIForName(
    "UniversityOfSheffield");
OInstance uosInstance = ontology.getOInstance(uosURI);

// worksFor property
OURI worksForURI = ontology.createOURIForName(
    "worksFor");
ObjectProperty worksFor = ontology.getObjectProperty(
    worksForURI);

// find all the people who work for the University of Sheffield
List<OResource> uniEmployees = 
    ontology.getOResourcesWith(worksFor, uosInstance);
\end{lstlisting}
\end{frame}

\begin{frame}
\frametitle{A note about URIs}
\begin{itemize}
\item Ontology resources are identified by URIs.
\item URI is treated as a \emph{namespace} (everything up to and including the
  last \#, / or :, in that order) and a \emph{resource name} (the rest)
\item Ontology LR provides factory methods to create \texttt{OURI} objects:
  \begin{itemize}
  \item \texttt{createOURI} takes a complete URI string
  \item \texttt{createOURIForName} takes the resource name and prepends the
    ontology LR's \emph{default namespace}
  \item \texttt{generateOURI} takes a resource name, prepends the default NS
    and adds a unique suffix.
  \end{itemize}
\item Only ASCII letters, numbers and certain symbols are permitted in URIs,
  other characters (including spaces) must be escaped.
  \begin{itemize}
  \item \texttt{OUtils} defines common escaping methods.
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Extending the ontology}
\begin{lstlisting}[texcl]
OURI personURI = ontology.createOURIForName("Person");
OClass personClass = ontology.getOClass(personURI);

// create a new class as a subclass of an existing class
OURI empURI = ontology.createOURIForName("Employee");
OClass empClass = ontology.addOClass(empURI);
personClass.addSubClass(empClass);

// create an instance
OURI fredURI = ontology.createOURIForName("FredSmith");
OInstance fred = ontology.addOInstance(fredURI,
                                       empClass);

// Fred works for the University of Sheffield
fred.addObjectPropertyValue(worksFor, uosInstance);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Exporting the ontology}
\begin{lstlisting}[texcl]
OutputStream out = ....
ontology.writeOntologyData(out,
    OConstants.OntologyFormat.RDFXML, false);
\end{lstlisting}
\begin{itemize}
\item \lstinline!false! means don't include \lstinline!OResource!s that came
  from an import (\lstinline!true! would embed the imported data in the
  exported ontology).
\item Other formats are \lstinline!TURTLE!, \lstinline!N3! and
  \lstinline!NTRIPLES!.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Ontology API in JAPE}
\begin{itemize}
\item Recall that JAPE RHS blocks have access to an \verb|ontology| parameter.
\item Can use JAPE rules for ontology \emph{population} or \emph{enrichment}
\item Create new instances or property values in an ontology based on patterns
  found in the text.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 1: Basic Ontology API}
\begin{itemize}
\item Start GATE Developer.
\item Load \verb|hands-on/ontology/exercise1.xgapp|
\item This xgapp loads two controllers.  ``Exercise 1 application'' is a
  ``trick'' application containing a JAPE grammar \verb|exercise1.jape| with a
  single rule that is guaranteed to fire exactly once when the application is
  run.
\item The application loads \verb|hands-on/ontology/demo.owl| and configures
  the JAPE transducer with that ontology.
\item We treat the RHS of the rule as a ``scratch pad'' to test Java code that
  uses the ontology API.
\item Also loads ``Reset ontology'' application you can use to reset the
  ontology to its original state.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 1: Basic Ontology API}
\begin{itemize}
\item The initial JAPE file contains comments giving some suggested tasks.
\item See how many of these ideas you can implement.
\item Each time you modify the JAPE file you will need to re-init the
  ``Exercise 1 transducer'' then run the ``Exercise 1 application''.
\item Open the ontology viewer to see the result of your changes.
\item You will need to close and re-open the viewer each time.
\item Use the reset application as necessary.
\end{itemize}
Remember: ontology API JavaDocs at
\url{http://gate.ac.uk/gate/doc/javadoc/?gate/creole/ontology/package-summary.html}
\end{frame}

\ifsolutions
\addtocounter{framenumber}{-2}\framenumbersfalse
\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 1: Solutions}
Possible solutions (exception handling omitted):
\begin{lstlisting}[texcl,basicstyle=\scriptsize\ttfamily]
// Create an instance of the City class representing Sheffield
OURI cityURI = ontology.createOURIForName("City");
OClass cityClass = ontology.getOClass(cityURI);
OURI sheffieldURI = ontology.generateOURI("Sheffield");
OInstance sheffield = ontology.addOInstance(sheffieldURI,
                                            cityClass);

// Create a new class named "University" as a subclass of Organization
OURI orgURI = ontology.createOURIForName("Organization");
OURI uniURI = ontology.createOURIForName("University");
OClass orgClass = ontology.getOClass(orgURI);
OClass uniClass = ontology.addOClass(uniURI);
orgClass.addSubClass(uniClass);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 1: Solutions (continued)}
\begin{lstlisting}[texcl,basicstyle=\scriptsize\ttfamily]
// Create an instance of the University class representing the University of Sheffield
OURI unishefURI = ontology.generateOURI(
    OUtils.toResourceName("University of Sheffield"));
OInstance unishef = ontology.addOInstance(unishefURI,
                                          uniClass);

// Create an object property basedAt with domain Organization and range Location
OURI locationURI = ontology.createOURIForName("Location");
OClass locationClass = ontology.getOClass(locationURI);
OURI basedAtURI = ontology.createOURIForName("basedAt");
ObjectProperty basedAt = ontology.addObjectProperty(
    basedAtURI, Collections.singleton(orgClass),
    Collections.singleton(locationClass));

// Specify that the University of Sheffield is basedAt Sheffield
unishef.addObjectPropertyValue(basedAt, sheffield);
\end{lstlisting}
\end{frame}
\framenumberstrue
\fi

\begin{frame}[fragile]
\frametitle{Ontology-aware JAPE}
\begin{itemize}
\item When supplied with an ontology parameter, JAPE can do ontology-aware
  matching.
\item In this mode the feature named ``class'' on an annotation is special: it
  is assumed to be an ontology class URI, and will match any subclass.
\item If the class feature is not a complete URI, it has the ontology's default
  namespace prepended.
  \begin{itemize}
  \item e.g. \lstinline!{Lookup.class == "Location"}! with our demo ontology
    would match Lookup annotations with any subclass of
    \lstinline[language=html]|http://www.owl-ontologies.com/unnamed.owl#Location|,
    in the class feature, including ``City'', ``Country'', etc.
  \end{itemize}
\item When an ontology parameter is \emph{not} specified, class is treated the
  same as any other feature (not the case prior to GATE 5.2).
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Ontology Population}
\begin{itemize}
\item Ontology population is the process of adding instances to an ontology
  based on information found in text.
\item We will explore a very simple example, real-world ontology population
  tasks are complex and domain-specific.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Ontology population example}
\begin{itemize}
\item The demo ontology from exercise 1 contains a ``Location'' class with
  subclasses ``City'', ``Country'', ``Province'' and ``Region''.
\item These correspond to subsets of the ANNIE named entities.
\item We want to populate our ontology with instances for each location in a
  document.
\item Very simple assumption -- if two Location annotations have the same text,
  they refer to the same location.
  \begin{itemize}
  \item Typically you would need to disambiguate, e.g. with coreference
    information.
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 2: Ontology population}
\begin{itemize}
\item Start GATE Developer
\item Load \verb|hands-on/ontology/exercise2.xgapp|
\item This xgapp again loads the demo ontology and defines the ontology reset
  controller.
\item Second controller in this case is a normal ANNIE with two additional JAPE
  grammars.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{ANNIE {\tt locType} to Ontology Class}
\begin{itemize}
\item ANNIE creates \texttt{Location} annotations with a \texttt{locType}
  feature, and \texttt{Organization} annotations with an \texttt{orgType}
  feature.
  \begin{itemize}
  \item e.g. locType = region
  \end{itemize}
\item The first of the two additional grammars (``NEs to Mentions'') creates
  annotations of type Mention with a ``class'' feature derived from the
  \texttt{locType} or \texttt{orgType}.
\item Location (or Organization) annotations without a locType (or orgType) are
  mapped to the top-level Location (Organization) class.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Populating the ontology}
\begin{itemize}
\item Given these Mention annotations, we can now populate the ontology.
\item We want to create one instance for each distinct entity.
\item Use the RDFS ``label'' annotation property to associate the instance with
  its text.
\item So for each Mention of a Location, we need to:
  \begin{itemize}
  \item determine which ontology class it is a mention of
  \item see if there is already an instance of this class with a matching
    label, and if not, create one, and
  \item store the URI of the relevant ontology instance on the Mention
    annotation.
  \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{\color{gate-dark-green} Exercise 2: Ontology population}
Over to you!
\begin{itemize}
\item Fill in \verb|hands-on/ontology/exercise2.jape| to implement this
  algorithm.
\item As before, you need to re-init the Exercise 2 transducer each time you
  edit the JAPE file.
\item Use the ``Reset ontology'' application to clean up the ontology between
  runs (though if you do it right it won't create extra instances if you run
  again without cleaning).
\end{itemize}
\end{frame}

\ifsolutions
\addtocounter{framenumber}{-3}\framenumbersfalse
\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 2: Solution}
A possible solution
\begin{lstlisting}[texcl,basicstyle=\scriptsize\ttfamily]
// Create some useful objects - rdfs:label property and a Literal for the covered text.
AnnotationProperty rdfsLabel = ontology.getAnnotationProperty(
    ontology.createOURI(OConstants.RDFS.LABEL));
Literal text = new Literal(stringFor(doc, locAnnots));

for(Annotation mention : locAnnots) {
  // determine the right class
  OURI classUri = ontology.createOURI(
      (String)m.getFeatures().get("class"));
  OClass clazz = ontology.getOClass(classUri);

  // get all existing instances of that class
  Set<OInstance> instances = ontology.getOInstances(clazz,
      OConstants.DIRECT_CLOSURE);
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 2: Solution (continued)}
\begin{lstlisting}[firstnumber=last,texcl,basicstyle=\scriptsize\ttfamily]
  // see if any of them have the right label -- if so, we assume they're the same
  OInstance inst = null;
  for(OInstance candidate : instances) {
    if(candidate.getAnnotationPropertyValues(
          rdfsLabel).contains(text)) {
      // found it!
      inst = candidate;
      break;
    }
  }
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\omitfromhandout
\frametitle{\color{gate-dark-green} Exercise 2: Solution (continued)}
\begin{lstlisting}[firstnumber=last,texcl,basicstyle=\scriptsize\ttfamily]
  if(inst == null) {
    // not found an existing instance, create one with a generated name
    String instName = OUtils.toResourceName(text.getValue());
    OURI instURI = ontology.generateOURI(instName + "_");
    inst = ontology.addOInstance(instURI, clazz);
    // and label it with the covered text
    inst.addAnnotationPropertyValue(rdfsLabel, text);
  }

  // finally, store the URI of the (new or existing) instance on the annotation
  mention.getFeatures().put("inst",
      inst.getONodeID().toString());
}
\end{lstlisting}
\end{frame}
\framenumberstrue
\fi

\begin{frame}
\frametitle{Conclusions and further reading}
\begin{itemize}
\item This is a good example of a case where utility classes are useful.
\item We have used this technique in other projects, e.g.
  \url{gate.ac.uk/sale/icsd09/sprat.pdf}
\item Lots of tutorial materials on ontologies, OWL, etc. available online.
\item For GATE, best references are the user guide and javadocs.
\end{itemize}
\end{frame}

\end{document}
