DeSR Dependency Parser

src/Token.cpp

/*
**  DeSR
**  src/Token.cpp
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of DeSR.
**
**  DeSR is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  DeSR is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#include "platform.h"

#include "Sentence.h"
#include "Corpus.h"

// standard
#include <iostream>
#include <iomanip>

// library
#include "conf/conf_bool.h"

using namespace std;
using namespace Parser;

namespace Tanl {

AttributeIndex* Attributes::emptyAttrIndex = new AttributeIndex();

std::ostream& operator <<(std::ostream& os, AttributeIndex const& ai)
{
  FOR_EACH (Tanl::Text::WordIndex, ai, it)
    os << '<' << it->first << ", " << it->second << "> ";;
  return os << std::endl;
}

ostream& operator <<(ostream& os, Token const& tok)
{
  os << "<tok";
  FOR_EACH(Attributes, tok.attributes, fit) {
    os << "\" " << (*fit).first << "=\"" << (*fit).second;
  }
  return os << '>';
}

void Token::print(ostream& os, int indent) const
{
  os << setw(indent) << ' ' << "<tok "
     << "form=\"" << attributes.get("FORM")
     << "\" lemma=\"" << attributes.get("LEMMA")
     << "\" cpos=\"" << attributes.get("CPOS")
     << "\" pos=\"" << attributes.get("POS")
     << "\" morph=\"" << attributes.get("FEATS")
     << "\">" << endl;
}

//======================================================================
// Token

#define CONLL08                 // FIXME: for the time being
#ifdef CONLL08
static char const* DeprelTag = "DEPREL";
static char const* LemmaTag = "SPLIT_LEMMA";
static char const* PosTag = "PPOSS";
#else
static char const* DeprelTag = "DEPREL";
static char const* LemmaTag = "LEMMA";
static char const* PosTag = "CPOS";
#endif

bool Token::isNoun(Language const* lang)
{
  string const* pos = get(PosTag);
  return pos && strStartsWith(pos->c_str(), lang->nounCPos);
}

bool Token::isPreposition(Language const* lang)
{
  string const* pos = get(PosTag);
  return pos && strStartsWith(pos->c_str(), lang->prepCPos);
}

bool Token::isTime(Language const* lang)
{
  // from gold deprel
  string const* label = ((Token*)this)->get(DeprelTag);
  return label && strStartsWith(label->c_str(),lang->timeDep);
}

bool Token::isLocation(Language const* lang)
{
  // from gold deprel
  string const* label = ((Token*)this)->get(DeprelTag);
  return label && strStartsWith(label->c_str(),lang->locDep);
}

bool Token::isVerb(Language const* lang)
{
  string const* pos = get(PosTag);
  return pos && strStartsWith(pos->c_str(), lang->verbCPos);
}

std::string const* Token::getLemma() const
{
  return get(LemmaTag);
}

std::string const* Token::getPos() const
{
  return get(PosTag);
}

//======================================================================
// TreeToken

TreeToken* TreeToken::ancestorCPos(char const* pos, Sentence& tokens)
{
  for (int nid = linkHead(); nid != 0; ) {
    TreeToken* tokn = (TreeToken*)tokens[nid-1];
    std::string const* attr = tokn->get("CPOS");
    if (attr && *attr == pos)
      return tokn;
    nid = tokn->linkHead();
  }
  return 0;
}

TreeToken* TreeToken::follow(Parser::TokenPath const& tp, Sentence& sentence)
{
  TreeToken* tok = this;
  FOR_EACH(vector<TokenPath::Direction>, tp.path, pit) {
    switch (*pit) {
    case TokenPath::leftChild:
      // move to the furthest left child (as in Nivre), if present
      tok = tok->left.empty() ? 0 : tok->left.front();
      break;

    case TokenPath::rightChild:
      // move to the furthest right child (as in Nivre), if present
      tok = tok->right.empty() ? 0 : tok->right.front();
      break;

    case TokenPath::parent:
      if (tok->linkHead() == 0) {
        tok = 0;
        break;
      }
      tok = sentence[tok->linkHead() - 1];
      break;

    case TokenPath::leftSibling: {
      if (tok->linkHead() == 0) {
        tok = 0;
        break;
      }
      TreeToken* parent = sentence[tok->linkHead() - 1];
      TreeToken* sibl = 0;
      // find token following tok in left children
      FOR_EACH(vector<TreeToken*>, parent->left, cit) {
        if (*cit == tok) {
          if (++cit != parent->left.end()) {
            sibl = *cit;
            break;
          }
        }
      }
      if (sibl) {
        tok = sibl;
        break;
      }
      // find token preceding tok in right children
      FOR_EACH(vector<TreeToken*>, parent->right, cit) {
        if (*cit == tok) {
          tok = 0;
          break;
        }
        sibl = *cit;
      }
      tok = (tok == 0) ? sibl : 0;
      break;
    }

    case TokenPath::rightSibling: {
      if (tok->linkHead() == 0) {
        tok = 0;
        break;
      }
      TreeToken* parent = sentence[tok->linkHead() - 1];
      TreeToken* sibl = 0;
      // find token following tok in right children
      FOR_EACH(vector<TreeToken*>, parent->right, cit) {
        if (*cit == tok) {
          if (++cit != parent->left.end()) {
            sibl = *cit;
            break;
          }
        }
      }
      if (sibl) {
        tok = sibl;
        break;
      }
      // find token preceding tok in left children
      FOR_EACH(vector<TreeToken*>, parent->left, cit) {
        if (*cit == tok) {
          tok = 0;
          break;
        }
        sibl = *cit;
      }
      tok = (tok == 0) ? sibl : 0;
      break;
    }

    case TokenPath::previous:
      tok =  (tok->id > 1) ? sentence[tok->id - 2] : 0;
      break;

    case TokenPath::next:
      tok = (tok->id < sentence.size()) ? sentence[tok->id] : 0;
      break;
    }

    if (!tok) break;
  }
  return tok;
}

void TreeToken::print(ostream& os, int indent) const
{
  Attributes& attributes = token->attributes;
  os << setw(indent) << ' ' << "<tok id=\"" << id
     << "\" form=\"" << attributes.get("FORM")
     << "\" lemma=\"" << attributes.get("LEMMA")
     << "\" cpos=\"" << attributes.get("CPOS")
     << "\" pos=\"" << attributes.get("POS")
     << "\" morph=\"" << attributes.get("FEATS")
     << "\" head=\"" << linkHead() << "\" deprel=\"" << linkLabel() << "\">" << endl;
  indent += 2;
  FOR_EACH (vector<TreeToken*>, left, it)
    (*it)->print(os, indent);
  FOR_EACH (vector<TreeToken*>, right, it)
    (*it)->print(os, indent);
}

void TreeToken::printTab(ostream& os)
{
  Attributes& attributes = token->attributes;
  bool first = true;
  FOR_EACH (vector<char const*>, attributes.attributeIndex->names, it) {
    string const* tag = attributes.get(*it);
    if (first)
      first = false;
    else
      os << "\t";
    os << ((!tag || tag->empty()) ? "_" : *tag);
  }
}

void TreeToken::printTab(ostream& os, Corpus const& corpus)
{
  Attributes& attributes = token->attributes;
  TokenFields::const_iterator tfit = corpus.tokenFields.begin();
  bool first = true;
  FOR_EACH (vector<char const*>, attributes.attributeIndex->names, it) {
    if (first)
      first = false;
    else
      os << "\t";
    switch (tfit->role) {
    case TokenField::head:
      os << linkHead(); break;
    case TokenField::deprel:
      os << (linkLabel().empty() ? tfit->default_ : linkLabel()); break;
    default:
      if (tfit->use == TokenField::ignore)
        os << tfit->default_;
      else {
        string const* tag = attributes.get(*it);
        os << ((!tag || tag->empty()) ? tfit->default_ : *tag);
    }
    }
    ++tfit;
  }
}

void TreeToken::printConll08(ostream& os, Corpus const& corpus, vector<int>* preds)
{
  Attributes& attributes = token->attributes;
  TokenFields::const_iterator tfit = corpus.tokenFields.begin();
  TokenLinks::const_iterator tlit = token->links.begin();
  bool first = true;
  FOR_EACH (vector<char const*>, attributes.attributeIndex->names, it) {
    string const* tag = attributes.get(*it);
    if (first)
      first = false;
    else
      os << "\t";
    if (!tfit->link.empty())
      os << tlit->head;
    else if (!tfit->label.empty()) {
      if (tlit != token->links.end()) {
        os << (tlit->label.empty() ? tfit->default_ : tlit->label);
        ++tlit;
      }
    } else {
      os << ((!tag || tag->empty()) ? tfit->default_ : *tag);
    }
    ++tfit;
  }
  // print links
# ifdef COMPACT
  preds = 0;
# endif
  if (preds) {
    unsigned predNo = 0;
    for (; tlit != token->links.end(); ++tlit) {
      int head = tlit->head;
      while (head != (*preds)[predNo++])
        os << "\t_";
      os << '\t' << tlit->label;
    }
    while (predNo++ < preds->size())
      os << "\t_";
  } else {
    for (; tlit != token->links.end(); ++tlit)
      os << '\t' << tlit->label << '[' << tlit->head << ']';
  }
}

} // namespace Tanl

 

Copyright © 2005-2007 G. Attardi. Generated on 13 Aug 2009 by doxygen 1.5.7.1.