DeSR Dependency Parser

src/Parser.cpp

/*
**  DeSR
**  src/Parser.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2006  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of DeSR.
**
**  DeSR is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  DeSR is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "Parser.h"
#include "ReviseEventStream.h"
#include "Corpus.h"
#include "version.h"

// library
#include "ixe/io/File.h"

// standard
#include <iostream>

using namespace std;
using namespace IXE;

#define MAX_LINE_LEN 8196

namespace Parser {

conf<string>    algorithm("Algorithm", "SVM"); // ME, AP, PA, SVM
conf<string>    lang("Language", "en"); // en, it, pt, sp, jp, ...
conf<int>       beam("Beam", 1, 1); // beam size

conf<int>       Parser::featureCutoff("FeatureCutoff", 0);
conf<bool>      Parser::verbose("Verbose", false);

float const GlobalInfo::freqRatio = 1.5;

std::map<char const*, ParserFactory*>& ParserMap::get()
{
  static std::map<char const*, ParserFactory*> parserFor;
  return parserFor;
}

ParserFactory* ParserMap::get(char const* type)
{
  std::map<char const*, ParserFactory*>& parserMap = get();
  map<char const*, ParserFactory*>::const_iterator rit = parserMap.find(type);
  return (rit == parserMap.end()) ? 0 : rit->second;
}

Parser* Parser::create(char const* modelFile)
{
  if (modelFile) {
    // read parameters from modelFile
    ifstream ifs(modelFile);
    if (!ifs) {
      cerr << "Missing model file: " << modelFile << endl;
      return 0;
    }
    ::Parser::Parser::readHeader(ifs);
    ifs.close();
  }
  // here we know which algorith to use
  ParserFactory* factory = ParserMap::get(algorithm->c_str());
  if (!factory) {
    cerr << "No such algorithm: " << *algorithm << endl;
    return 0;
  }
  ::Parser::Parser* parser = factory(modelFile);
  if (parser == 0) {
    cerr << "Could not load: " << modelFile << endl;
    return 0;
  }
  return parser;
}

Enumerator<Sentence*>* Parser::pipe(Enumerator<std::vector<Token*>*>& tce)
{
  return new ParserPipe(*this, tce);
}

Enumerator<Sentence*>* Parser::pipe(Enumerator<Sentence*>& tce)
{
  return new ParserSentPipe(*this, tce);
}

void Parser::parse(SentenceReader* reader, ostream& os)
{
  int las = 0;
  int uas = 0;
  int tot = 0;
  int sent = 0;
# ifdef _WIN32
  if (os == cout)
    _setmode(_fileno(stdout), _O_BINARY); // no \r
# endif

  while (reader->MoveNext()) {
    Sentence* sin = reader->Current();
    Sentence* sout = parse(sin);
    reader->corpus->print(os, *sout);
    os << endl;
    ++sent;
    // check accuracy
    Sentence::const_iterator soutIt = sout->begin();
    FOR_EACH (Sentence, *sin, sinIt) {
      TreeToken* tin = *sinIt;
      TreeToken* tout = *soutIt++;
      tot++;
      if (tin->linkHead() == tout->linkHead()) {
        uas++;
        if (tin->linkLabel() == tout->linkLabel())
          las++;
      }
    }
    delete sin;
    delete sout;
  }
  showEval(tot, las, uas, sent);
}

void Parser::writeHeader(ostream& os)
{
  os << "<desr alg='" << *algorithm << "' version='" << version << "'>" << endl;
  Configuration::Map& vars = Configuration::variables();
  FOR_EACH (Configuration::Map, vars, vit)
    vit->second->serialize(os);
  os << "</desr>" << endl;
}

bool Parser::readHeader(istream& is)
{
  Configuration::reset();
  Configuration::load(is);
  return true;
}

void GlobalInfo::extract(Sentence const& sentence)
{
  Language const* lang = sentence.language;
  // look for noun child of preposition with time/loc dependency
  FOR_EACH (Sentence, sentence, sit) {
    TreeToken* node = *sit;
    Token& tok = *node->token;
    int head = node->linkHead();
    if (head == 0)
      continue;
    if (tok.isNoun(lang)) {
      string const* noun = tok.getLemma();
      if (noun && !noun->empty()) {
        if (tok.isTime(lang)) {
          // add to time lemmas
          timeLemmas.add(*noun);
        } else if (tok.isLocation(lang)) {
          // add to location lemmas
          locLemmas.add(*noun);
        }
        // sometimes the link is indirect
        Token* par = sentence[head - 1]->token;
        if (par->isPreposition(lang)) {
          if (par->isTime(lang)) {
            // add to time lemmas
            timeLemmas.add(*noun);
          } else if (par->isLocation(lang)) {
            // add to location lemmas
            locLemmas.add(*noun);
          }
        }
      }
    }
  }
}

void GlobalInfo::clearRareEntities()
{
  for (WordCounts::iterator pit = timeLemmas.begin();
       pit != timeLemmas.end(); ) {
    WordCounts::iterator cur = pit++;
    int tc = cur->second;
    int lc = locLemmas.count(cur->first);
    if (tc >= freqRatio * lc)
      locLemmas.erase(cur->first);
    else if (lc >= freqRatio * tc)
      timeLemmas.erase(cur);
  }
}

void GlobalInfo::save(ofstream& ofs)
{
  // dump timeLemmas
  ofs << timeLemmas.size() << endl;
  FOR_EACH (WordCounts, timeLemmas, pit)
    ofs << pit->first << endl;
  // dump locLemmas
  ofs << locLemmas.size() << endl;
  FOR_EACH (WordCounts, locLemmas, pit)
    ofs << pit->first << endl;
}

void GlobalInfo::load(ifstream& ifs)
{
  char line[MAX_LINE_LEN];
  // read timeLemmas
  if (ifs.getline(line, MAX_LINE_LEN)) {
    int n = atoi(line);         // number of timeLemmas
    while (n-- && ifs.getline(line, MAX_LINE_LEN))
      timeLemmas.add(line);
    // read locLemmas
    if (ifs.getline(line, MAX_LINE_LEN)) {
      n = atoi(line);           // number of locLemmas
      while (n-- && ifs.getline(line, MAX_LINE_LEN))
        locLemmas.add(line);
    }
  }
}

void Parser::showEval(int tokenCount, int las, int uas, int sentCount)
{
  cerr << "UAS: " << float(uas)/tokenCount << " (" << uas << '/' << tokenCount << ')' << endl;
  cerr << "LAS: " << float(las)/tokenCount << " (" << las << '/' << tokenCount << ')' << endl;
  cerr << "Sentences: " << sentCount << endl;
}

// ======================================================================

ParserPipe::ParserPipe(Parser& parser, Enumerator<std::vector<Token*>*>& tve) :
  parser(parser),
  tve(tve),
  language(Language::get(lang->c_str())) // FIXME: should not use globals
{
}

bool ParserPipe::MoveNext()
{
  return tve.MoveNext();
}

Sentence* ParserPipe::Current()
{
  vector<Token*>* sent = tve.Current();
  Sentence* sentence = new Sentence(language);
  int id = 1;
  FOR_EACH (vector<Token*>, *sent, vit) {
    Token* tok = *vit;
    TreeToken* token = new TreeToken(id++, tok->form, tok->attributes, tok->links);
    sentence->push_back(token);
    delete tok;
  }
  delete sent;
  return parser.parse(sentence);
}

// ======================================================================

ParserSentPipe::ParserSentPipe(Parser& parser, Enumerator<Sentence*>& tve) :
  parser(parser),
  tve(tve)
{ }

bool ParserSentPipe::MoveNext()
{
  return tve.MoveNext();
}

Sentence* ParserSentPipe::Current()
{
  Sentence* sentence = tve.Current();
  return parser.parse(sentence);
}

} // namespace Parser

 

Copyright © 2005-2007 G. Attardi. Generated on 13 Aug 2009 by doxygen 1.5.7.1.