DeSR Dependency Parser

src/Token.h

/*
**  DeSR
**  src/Token.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2005 Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of DeSR.
**
**  DeSR is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  DeSR is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program. If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef DeSR_Token_H
#define DeSR_Token_H

// standard
#include <string>
#include <vector>

// library
#include "text/WordIndex.h"
#include "Common/util.h"        // FOR_EACH

//local
#include "Language.h"
#include "TokenPath.h"

namespace Tanl {

class Sentence;
class Corpus;

typedef unsigned        AttributeId;

struct AttributeIndex : public Text::WordIndex
{
  std::vector<char const*>      names;

  AttributeId   insert(const char* name) {
    AttributeId id = Tanl::Text::WordIndex::insert(name);
    if (id >= names.size()) {
      // add new name to names
      names.resize(id + 1);
      names[id] = ::strdup(name);
    }
    return id;
  }

  AttributeIndex& operator =(AttributeIndex const& ai) {
    Tanl::Text::WordIndex::operator=(ai);
    for (unsigned i = 0; i < names.size(); i++)
      free((void*)names[i]);
    names.resize(ai.size());
    for (unsigned i = 0; i < ai.names.size(); i++)
      names[i] = strdup(ai.names[i]);
    return *this;
  }

  ~AttributeIndex() {
    for (unsigned i = 0; i < names.size(); i++)
      free((void*)names[i]);
  }

};

std::ostream& operator <<(std::ostream& os, AttributeIndex const& ai);

struct Attributes
{
  typedef std::string           Attribute;

  AttributeIndex*               attributeIndex;
  std::vector<Attribute>        values;

  static AttributeIndex*        emptyAttrIndex;

  Attributes(AttributeIndex* attributeIndex) :
    attributeIndex(attributeIndex)
  {
    if (attributeIndex)
      values.resize(attributeIndex->size());
    else
      this->attributeIndex = emptyAttrIndex;
  }

  Attributes(AttributeIndex* attributeIndex, std::vector<Attribute>& values) :
    attributeIndex(attributeIndex),
    values(values)
  { }

  Attribute& operator [](int i) { return values[i]; }

  struct const_iterator {
    const_iterator(Attributes const& attributes, int fit = 0) :
      attributes(attributes),
      fit(fit)
    { }
      
    bool operator ==(const const_iterator& other) {
      return &attributes == &other.attributes && fit == other.fit;
    }
    bool operator !=(const const_iterator& other) { return !(*this == other); }
    const_iterator& operator ++() { fit++; return *this; }
    const_iterator operator ++(int) { // postincrement
      const_iterator tmp = *this;
      ++*this;
      return tmp;
    }

    // can't use string& since pair<> constructor uses reference paramaters.
    std::pair<char const*, std::string const*> operator*() {
      std::string const& val = attributes.values[fit];
      return std::make_pair(attributes.attributeIndex->names[fit], &val);
    }

    Attributes const& attributes;
    int fit;
  };

  const_iterator begin() const { return const_iterator(*this); }
  const_iterator end() const { return const_iterator(*this, values.size()); }

  std::string const* get(std::string& name) const {
    AttributeId id = attributeIndex->index(name.c_str());
    return (id == Tanl::Text::WordIndex::None) ? 0 : &values[id];
  }

  std::string const* get(char const* name) const {
    AttributeId id = attributeIndex->index(name);
    return (id == Tanl::Text::WordIndex::None) ? 0 : &values[id];
  }

  AttributeId   index(char const* key) const {
    return attributeIndex->index(key);
  }

  void insert(char const* key, char const* value) {
    AttributeId id = attributeIndex->insert(key); // get an ID for key
    values.resize(id + 1);
    values[id] = value;
  }

  void insert(char const* key, std::string const& value) {
    AttributeId id = attributeIndex->insert(key); // get an ID for key
    values.resize(id + 1);
    values[id] = value;
  }
};

struct TokenLink
{
  TokenLink(int head = -1, char const* label = "") :
    head(head),
    label(label)
  { }

  int           head;
  std::string   label;
};

typedef std::vector<TokenLink> TokenLinks;

struct Token
{

  struct Morpho {
    std::string         morph;  
    char                Case;   
    char                gender; 
    char                mode;   
    char                negative; 
    char                number; 
    char                person; 
    char                tense;  
    char                transitive; 
    char                extra[20]; 

    Morpho(const char* morph = "", size_t morphSize = 0) :
      morph(morph, morphSize),
      Case(0),
      gender(0),
      mode(0),
      negative(0),
      number(0),
      person(0),
      tense(0),
      transitive(0)
    {
      extra[0] = '\0';
    }

    void set(MorphExtractor::Features& mf) {
      Case = mf.Case[0];
      gender = mf.gender[0];
      number = mf.number[0];
      person = mf.person[0];
      mode = mf.mode[0];
      negative = mf.negative[0];
      number = mf.number[0];
      person = mf.person[0];
      tense = mf.tense[0];
      transitive = mf.trans[0];
      int extraLen = sizeof(extra) - 1;
      strncpy(extra, mf.extra, extraLen);
      extra[extraLen] = '\0';
    }
  };

  Token(std::string form, Attributes& attributes) :
    form(form),
    attributes(attributes),
    links(1, TokenLink(-1))
  { }

  Token(std::string form, Attributes& attributes, TokenLinks& links) :
    form(form),
    attributes(attributes),
    links(links)
  { }

  Token(Attributes& attributes) :
    attributes(attributes),
    links(1, TokenLink(-1))
  { }

  Token(std::string& form, AttributeIndex* attributeIndex) :
    form(form),
    attributes(attributeIndex),
    links(1, TokenLink(-1))
  { }

  Token(char const* form, AttributeIndex* attributeIndex) :
    form(form),
    attributes(attributeIndex),
    links(1, TokenLink(-1))
  { }

  Token(AttributeIndex* attributeIndex) :
    attributes(attributeIndex),
    links(1, TokenLink(-1))
  { }

  std::string const*    get(char const* key) const {
    return attributes.get(key);
  }

  void          set(char const* key, char const* value) {
    attributes.insert(key, value);
  }

  void          set(char const* key, std::string const& value) {
    attributes.insert(key, value);
  }

  int           attrIndex(char const* name) const {
    std::vector<char const*> const& names = attributes.attributeIndex->names;
    /*
    std::vector<char const*>::iterator it = find_if(names.begin(), names.end(),
                                               std::bind2nd(Tanl::Text::eqstr(), name));
    return (it == names.end()) ? -1 : it - names.begin();
    */
    unsigned idx = 0;
    FOR_EACH (std::vector<char const*>, names, it) {
      if (!::strcmp(*it, name))
        break;
      idx++;
    }
    return (idx >= names.size()) ? -1 : idx;
  }

  bool          isNoun(Language const* lang);
  bool          isPreposition(Language const* lang);
  bool          isTime(Language const* lang);
  bool          isLocation(Language const* lang);
  bool          isVerb(Language const* lang);
  std::string const*    getLemma() const; // FIXME: should depend on corpus
  std::string const*    getPos() const; // FIXME: should depend on corpus

  void          print(std::ostream& os, int indent = 0) const;

  std::string   form;           
  Morpho        morpho;         
  Attributes    attributes;     
  TokenLinks    links;          

};

std::ostream& operator <<(std::ostream& os, Token const& tok);

struct TreeToken
{
  TreeToken(std::string& form, AttributeIndex* ai) :
    token(new Token(form, ai))
  { }

  TreeToken(int id, char const* form, AttributeIndex* attributeIndex = 0) :
    token(new Token(form, attributeIndex)),
    id(id)
  { }

  TreeToken(int id, std::string& form, Attributes& attributes,
            TokenLinks& links) :
    token(new Token(form, attributes, links)),
    id(id)
  { }

  TreeToken(int id, std::string& form, Attributes& attributes) :
    token(new Token(form, attributes)),
    id(id)
  { }

  TreeToken(int id, Token* token) :
    token(token),
    id(id)
  { } 

  TreeToken(TreeToken const& tok) :
    token(new Token(*tok.token)),
    id(tok.id),
    left(tok.left),
    right(tok.right)
  { }

  virtual ~TreeToken() { delete token; }

  Token*                        token;
  unsigned                      id;
  std::vector<TreeToken*>       left; 
  std::vector<TreeToken*>       right; 

  std::string const*    get(char const* key) const {
    // FiIXME: should work for all OUTPUT attributes:
    return (strcmp(key, "DEPREL")) ? token->attributes.get(key) : &linkLabel();
  }

  void          set(char const* key, char const* value) {
    token->set(key, value);
  }

  void          set(char const* key, std::string const& value) {
    token->set(key, value);
  }

  int                   linkHead(unsigned i = 0) const {
    return token->links[i].head;
  }
  int                   linkHead(int h, unsigned i = 0) {
    return token->links[i].head = h;
  }
  std::string const&    linkLabel(unsigned i = 0) const {
    return token->links[i].label;
  }
  std::string const&    linkLabel(std::string& l, unsigned i = 0) {
    return token->links[i].label = l;
  }
  std::string const&    linkLabel(char const* l, unsigned i = 0) {
    return token->links[i].label = l;
  }

  int   size() {
    int count = 0;
    FOR_EACH(std::vector<TreeToken*>, left, it)
      count += (*it)->size();
    FOR_EACH(std::vector<TreeToken*>, right, it)
      count += (*it)->size();
    return count + 1;
  }

  TreeToken* follow(Parser::TokenPath const& tp, Sentence& sentence);

  int   leftmostDescendant() {
    int lmd = id;
    FOR_EACH(std::vector<TreeToken*>, left, it)
      lmd = MIN(lmd, (*it)->leftmostDescendant());
    return lmd;
  }

  int   rightmostDescendant() {
    int lmd = id;
    FOR_EACH(std::vector<TreeToken*>, right, it)
      lmd = MAX(lmd, (*it)->rightmostDescendant());
    return lmd;
  }

  TreeToken*    ancestorCPos(char const* pos, Sentence& sent);

  TreeToken*    descendantCPos(char const* pos) {
    // start from children
    FOR_EACH(std::vector<TreeToken*>, left, it) {
      TreeToken* tokn = *it;
      std::string const* attr = tokn->get("CPOS");
      if (attr && *attr == pos)
        return tokn;
      tokn = tokn->descendantCPos(pos);
      if (tokn)
        return tokn;
    }
    FOR_EACH(std::vector<TreeToken*>, right, it) {
      TreeToken* tokn = *it;
      std::string const* attr = tokn->get("CPOS");
      if (attr && *attr == pos)
        return tokn;
      tokn = tokn->descendantCPos(pos);
      if (tokn)
        return tokn;
    }
    return 0;
  }

  // needed because of overloaded alternatives
  void          print(std::ostream& os, int indent = 0) const;

  void          printTab(std::ostream& os);

  void          printTab(std::ostream& os, Corpus const& corpus);

  void          printConll08(std::ostream& os, Corpus const& corpus,
                             std::vector<int>* preds = 0);
};

} // namespace Tanl

#endif // DeSR_Token_H

 

Copyright © 2005-2007 G. Attardi. Generated on 13 Aug 2009 by doxygen 1.5.7.1.