/*
 * Copyright (C) 2021 Maneesh P M <manu.pm55@gmail.com>
 * Copyright (C) 2018-2021 Matthieu Gautier <mgautier@kymeria.fr>
 * Copyright (C) 2011 Emmanuel Engelhart <kelson@kiwix.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU  General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301, USA.
 */

#include "xapianIndexer.h"
#include "libzim-resources.h"
#include "fs.h"
#include "tools.h"
#include "../constants.h"
#include <zim/error.h>
#include <sstream>
#include <fstream>
#include <stdexcept>
#include <cassert>

using namespace zim::writer;

/* Constructor */
XapianIndexer::XapianIndexer(const std::string& indexPath, const std::string& language, IndexingMode indexingMode, const bool verbose)
    : indexPath(indexPath),
      language(language),
      indexingMode(indexingMode)
{
  /* Build ICU Local object to retrieve ISO-639 language code (from
     ISO-639-3) */
  icu::Locale languageLocale(language.c_str());
  stemmer_language = languageLocale.getLanguage();

  /* Read the stopwords */
  std::string stopWord;
  try {
    this->stopwords = getResource("stopwords/" + language);
  } catch(ResourceNotFound& e) {}
  std::istringstream file(this->stopwords);
  while (std::getline(file, stopWord, '\n')) {
    this->stopper.add(stopWord);
  }
}

XapianIndexer::~XapianIndexer()
{
  if (!indexPath.empty()) {
    try {
#ifndef _WIN32
//[TODO] Implement remove for windows
      zim::DEFAULTFS::remove(indexPath + ".tmp");
      zim::DEFAULTFS::remove(indexPath);
#endif
    } catch (...) {
      /* Do not raise */
    }
  }
}

/*
 * `valuesmap` is a metadata associated with the Xapian database. We are using it
 * to attach slot numbers of each document in the index to the value they are storing.
 * These values and slot numbers are used in collapsing, filtering etc.
 *
 * Title index:
 * Slot 0: Title of the article. Used in collapsing articles with same name.
 * Slot 1: path/redirectPath of the article. Used in collapsing duplicates(redirects).
 *
 * Fulltext Index:
 * Slot 0: Title of the article. Used in collapsing articles with same name.
 * Slot 1: Word count of the article.
 * Slot 2: Geo position of the article. Used for geo-filtering.
 *
 * `kind` metadata indicate whether the database is a title or a fulltext index.
 *
 * `data` metadata indicate the type of data stored in the index. A value of "fullPath"
 * means the data stores the complete path with a namespace.
 */

void XapianIndexer::indexingPrelude()
{
  writableDatabase = Xapian::WritableDatabase(indexPath + ".tmp", Xapian::DB_CREATE_OR_OVERWRITE | Xapian::DB_NO_TERMLIST);

  switch (indexingMode) {
    case IndexingMode::TITLE:
      writableDatabase.set_metadata("valuesmap", "title:0;targetPath:1");
      writableDatabase.set_metadata("kind", "title");
      writableDatabase.set_metadata("data", "fullPath");
      break;
    case IndexingMode::FULL:
      writableDatabase.set_metadata("valuesmap", "title:0;wordcount:1;geo.position:2");
      writableDatabase.set_metadata("kind", "fulltext");
      writableDatabase.set_metadata("data", "fullPath");
      break;
  }
  writableDatabase.set_metadata("language", language);
  writableDatabase.set_metadata("stopwords", stopwords);
}

namespace
{

size_t getTermCount(const Xapian::Document& d)
{
  return std::distance(d.termlist_begin(), d.termlist_end());
}

size_t sizeOfIndexedText(const Xapian::Document& d)
{
  size_t n = 0;
  for (auto termIt = d.termlist_begin(); termIt != d.termlist_end(); ++termIt) {
    const std::string& term = *termIt;
    if ( term[0] != 'Z' ) {
      n += termIt.get_wdf() * term.size();
    }
  }
  return n;
}

} // unnamed namespace

/*
 * For title index, index the title with the full path (including the
 * namespace) as data of the document. The targetPath in valuesmap will store
 * the path without namespace.
 *
 * An exception is thrown if the total size of non-indexable text in the title
 * exceeds MAX_INDEXABLE_TITLE_WORD_SIZE (this is intended to detect omission
 * of too long words during indexing but may be triggered by excessive
 * whitespace and/or punctuation as well).
 *
 * TODO:
 * Currently for title index we are storing path twice (redirectPath/path in
 * valuesmap and path in index data). In the future, we want to keep only one of
 * these(index data if possible) to reduce index size while supporting the
 * collapse on path feature.
 */

void XapianIndexer::indexTitle(const std::string& path, const std::string& title, const std::string& targetPath)
{
  const size_t MAX_WORD_LENGTH = MAX_INDEXABLE_TITLE_WORD_SIZE;

  assert(indexingMode == IndexingMode::TITLE);
  Xapian::Stem stemmer;
  Xapian::TermGenerator indexer;
  indexer.set_max_word_length(MAX_WORD_LENGTH);
  indexer.set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM);
  try {
    stemmer = Xapian::Stem(stemmer_language);
    indexer.set_stemmer(stemmer);
    indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
  } catch (...) {}
  Xapian::Document currentDocument;
  currentDocument.clear_values();

  std::string fullPath = "C/" + path;
  currentDocument.set_data(fullPath);
  indexer.set_document(currentDocument);

  std::string unaccentedTitle = zim::removeAccents(title);

  currentDocument.add_value(0, title);
  if (targetPath.empty()) {
    currentDocument.add_value(1, path);
  } else {
    currentDocument.add_value(1, targetPath);
  }

  if (!unaccentedTitle.empty()) {
    std::string anchoredTitle = ANCHOR_TERM + unaccentedTitle;
    indexer.index_text(anchoredTitle, 1);
    if ( anchoredTitle.size() >= sizeOfIndexedText(currentDocument) + MAX_WORD_LENGTH ) {
      throw zim::TitleIndexingError("Too much loss of data during title indexing");
    }
    if ( getTermCount(currentDocument) == 1 ) {
      // only ANCHOR_TERM was added, hence unaccentedTitle is made solely of
      // non-word characters. Then add entire title as a single term.
      currentDocument.remove_term(*currentDocument.termlist_begin());
      if ( unaccentedTitle.size() <= MAX_WORD_LENGTH ) {
        currentDocument.add_term(unaccentedTitle);
      }
    }
  }

  /* add to the database */
  writableDatabase.add_document(currentDocument);
  empty = false;
}

void XapianIndexer::indexingPostlude()
{
  this->writableDatabase.commit();
  this->writableDatabase.compact(indexPath, Xapian::DBCOMPACT_SINGLE_FILE|Xapian::Compactor::FULL);
  this->writableDatabase.close();
}

