task.ssgrammar.hpp

Go to the documentation of this file.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use these files except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright 2012 - Gonzalo Iglesias, Adrià de Gispert, William Byrne

#ifndef SENTENCESPECIFICGRAMMARTASK_HPP
#define SENTENCESPECIFICGRAMMARTASK_HPP

namespace ucam {
namespace hifst {

template <class Data>
class SentenceSpecificGrammarTask: public ucam::util::TaskInterface<Data> {

  //Private variables are shown here. Private methods go after public methods
 private:

  ucam::util::IntegerPatternAddress ssgrammarfile_;

  bool addoovs_;
  bool deleteoovs_;
  std::size_t oovindex_;

  unsigned rule_id_offset_;

  SentenceSpecificGrammarData ssgd_;

 public:
  SentenceSpecificGrammarTask ( const ucam::util::RegistryPO& rg ) :
    rule_id_offset_ ( 0 ),
    ssgrammarfile_ ( rg.get<std::string> ( HifstConstants::kSsgrammarStore ) ),
    addoovs_ ( rg.getBool ( HifstConstants::kSsgrammarAddoovsEnable ) ) ,
    oovindex_ ( 0 ),
    deleteoovs_ ( rg.getBool (
                    HifstConstants::kSsgrammarAddoovsSourcedeletions ) ) {
    LDEBUG ( "Constructor done!" );
  };

  bool run ( Data& d ) {
    LDEBUG ( "HP size=" << d.hpinstances.size() );
    USER_CHECK ( d.grammar, "Big grammar data not available!" );
    oovindex_ = 1;
    d.ssgd = &ssgd_;
    LDEBUG ( "HP size=" << d.hpinstances.size() );
    d.stats->setTimeStart ("ssgrammar-extract");
    get ( d );
    d.stats->setTimeEnd ("ssgrammar-extract");
    LDEBUG ( "HP size=" << d.hpinstances.size() );
    // addFeedbackRules
    if ( ssgrammarfile_ ( d.sidx ) != "" )
      writessgrammar ( ssgrammarfile_ ( d.sidx ) );
    LDEBUG ( "Finished run method" );
    return false;
  };

 private:

  std::size_t createOOVRule ( const std::string& oov ) {
    std::size_t newindex = ssgd_.grammar->sizeofvpos + ( rule_id_offset_++ );
    unsigned n = ucam::util::toNumber<unsigned> ( oov );
    if ( n >= OOVID && !deleteoovs_ )
      ssgd_.extrarules[ newindex ] = "X " + oov + " " + oov + " 0";
    else
      ssgd_.extrarules[ newindex  ] = "X " + oov + " <oov> 0";
    LINFO ( "New oov rule id=" << newindex << ",rule=" <<
            ssgd_.extrarules[ newindex ] );
    return newindex;
  }

  void writessgrammar ( const std::string& filename ) {
    FORCELINFO ( "Saving ssgrammar to " << filename );
    unordered_set<unsigned> seenrules;
    ucam::util::oszfstream o ( filename );
    for ( ssgrammar_rulesmap_t::iterator itx = ssgd_.rulesWithRhsSpan1.begin();
          itx != ssgd_.rulesWithRhsSpan1.end();
          ++itx ) {
      for ( ssgrammar_firstelementmap_t::iterator itx2 = itx->second.begin();
            itx2 != itx->second.end();
            ++itx2 ) {
        for ( unsigned k = 0; k < itx2->second.size(); ++k ) {
          if ( seenrules.find ( itx2->second[k] ) != seenrules.end() ) continue;
          seenrules.insert ( itx2->second[k] );
          o << ssgd_.grammar->getRule ( itx2->second[k] ) << endl;
        }
      }
    }
    for ( ssgrammar_rulesmap_t::iterator itx =
            ssgd_.rulesWithRhsSpan2OrMore.begin();
          itx != ssgd_.rulesWithRhsSpan2OrMore.end();
          ++itx ) {
      for ( ssgrammar_firstelementmap_t::iterator itx2 = itx->second.begin();
            itx2 != itx->second.end();
            ++itx2 ) {
        for ( unsigned k = 0; k < itx2->second.size(); ++k ) {
          if ( seenrules.find ( itx2->second[k] ) != seenrules.end() ) continue;
          seenrules.insert ( itx2->second[k] );
          o << ssgd_.grammar->getRule ( itx2->second[k] ) << endl;
        }
      }
    }
    o.close();
  };

  void get ( Data& d ) {
    ssgrammar_instancemap_t& hpinstances = d.hpinstances;
    ssgd_.reset();
    ssgd_.grammar = d.grammar;
    for ( ssgrammar_instancemap_t::iterator itx = hpinstances.begin();
          itx != hpinstances.end(); ++itx ) {
      LDEBUG ( "Search for [" << itx->first << "]" );
      std::string needle = itx->first + " ";
      int pos = exists ( needle );
      if ( -1 == pos ) {
        if ( addoovs_ )
          if ( phraseIsTerminalWord ( itx->first ) ) {
            std::size_t ruleid = createOOVRule ( itx->first );
            for ( unsigned k = 0; k < itx->second.size(); ++k ) {
              unsigned& x = itx->second[k].first;
              ssgd_.rulesWithRhsSpan1[x][itx->first].push_back ( ruleid );
              LDEBUG ( "***Adding (OOV) rule index " << ruleid << ":" << ssgd_.getRule (
                         ruleid ) );
            }
          }
        LDEBUG ( "Pattern not found!" );
        continue;
      }
      USER_CHECK ( pos >= 0, "positive value required!" );
      LDEBUG ( "Extracting indices for =>" << itx->first << ",size of pattern=" <<
               getSize ( itx->first ) <<
               ", number of instances at which this was found: (x,span): " <<
               itx->second.size() );
      unordered_set<unsigned> seenx;
      if ( getSize ( itx->first ) == 1 ) {
        for ( unsigned k = 0; k < itx->second.size(); ++k ) {
          unsigned& x = itx->second[k].first;
          if ( seenx.find ( x ) != seenx.end() ) {
            LDEBUG ( "Repeated:" << itx->first << " at x=" << x );
            continue;
          }
          seenx.insert ( x );
          LDEBUG ( "*calling addRuleIndicesRHS (1) at x=" << x );
          addRuleIndicesRHS ( needle, pos, ssgd_.rulesWithRhsSpan1[x] , d.tvcb );
          LDEBUG ( "*Done!" );
        }
      } else {
        for ( unsigned k = 0; k < itx->second.size(); ++k ) {
          unsigned& x = itx->second[k].first;
          if ( seenx.find ( x ) != seenx.end() ) {
            LDEBUG ( "Repeated:" << itx->first << " at x=" << x );
            continue;
          }
          seenx.insert ( x );
          LDEBUG ( "*calling addRuleIndicesRHS (2) at x=" << x );
          addRuleIndicesRHS ( needle, pos, ssgd_.rulesWithRhsSpan2OrMore[x] , d.tvcb );
          LDEBUG ( "*Done" );
        }
      }
      LDEBUG ( "Finished extracting indices for " << itx->first );
    }
    LDEBUG ( "Finished get method" );
  };

  void addRuleIndicesRHS ( const std::string& needle
                           , const int pos, ssgrammar_firstelementmap_t& rules
                           , const unordered_set<std::string>& vcb ) {
    USER_CHECK ( pos >= 0, "pos needs to be positive" );
    LDEBUG ( "**Adding indices for rules" );
    const GrammarData& g = *ssgd_.grammar;
    for ( unsigned j = pos; j < g.sizeofvpos ; ++j ) {
      if ( g.ct->ncompare ( needle.c_str(), g.filecontents.c_str() + g.vpos[j].p,
                            needle.size() ) ) break;
      if ( !g.isAcceptedByVocabulary ( j, vcb ) ) {
        LDEBUG ( "skipping rule (rejected by vcb):" << g.getRule ( j ) );
        continue;
      }
      std::string firstelement = g.getRHSSource ( j , 0 );
      getFilteredNonTerminal ( firstelement );
      LDEBUG ( "***Adding rule #" << j << ":" << g.getRule ( j ) );
      rules[firstelement].push_back ( j );
    }
    if ( pos == 0 )  return;
    for ( int j = pos - 1; j >= 0 ; --j ) {
      if ( g.ct->ncompare ( needle.c_str(), g.filecontents.c_str() + g.vpos[j].p,
                            needle.size() ) ) break;
      if ( !g.isAcceptedByVocabulary ( j, vcb ) ) continue;
      std::string firstelement = g.getRHSSource ( j , 0 );
      getFilteredNonTerminal ( firstelement );
      LDEBUG ( "***Adding rules # " << j << ":" << g.getRule ( j ) );
      rules[firstelement].push_back ( j );
    }
  };

  inline int exists ( const std::string& needle ) {
    USER_CHECK ( needle.at ( needle.size() - 1 ) == ' ',
                 "This method requires a space appended to que queried string" );
    const GrammarData& g = *ssgd_.grammar;
    int oldmid = -1, mid = 0;
    int first = 0;
    int last = g.sizeofvpos - 1;
    //good old fashioned bs over the raw sequence of chars.
    while ( first <= last ) {
      mid = ( first + last ) / 2;
      if ( mid == oldmid ) break;
      int res = g.ct->ncompare ( needle.c_str(),
                                 g.filecontents.c_str() + g.vpos[mid].p, needle.size() );
      if ( res < 0 ) first = mid + 1;
      else if ( res > 0 ) last = mid - 1;
      else first = last + 1;
      oldmid = mid;
    }
    if ( !g.ct->ncompare ( needle.c_str(), g.filecontents.c_str() + g.vpos[mid].p,
                           needle.size() ) ) {
      return mid;
    }
    LDEBUG ( "Could not find: [" << needle << "]" );
    return -1;
  };

  ZDISALLOW_COPY_AND_ASSIGN ( SentenceSpecificGrammarTask );

};

}
}   // end namespaces

#endif