package com.nexwave.nquindexer;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.*;
import java.io.StringReader;

// specific dita ot
import com.nexwave.nsidita.DocFileInfo;

//Stemmers
import com.nexwave.stemmer.snowball.SnowballStemmer;
import com.nexwave.stemmer.snowball.ext.EnglishStemmer;
import com.nexwave.stemmer.snowball.ext.FrenchStemmer;
import com.nexwave.stemmer.snowball.ext.GermanStemmer;

//client-side support is yet to come for these stemmers
import com.nexwave.stemmer.snowball.ext.danishStemmer;
import com.nexwave.stemmer.snowball.ext.dutchStemmer;
import com.nexwave.stemmer.snowball.ext.finnishStemmer;
import com.nexwave.stemmer.snowball.ext.hungarianStemmer;
import com.nexwave.stemmer.snowball.ext.italianStemmer;
import com.nexwave.stemmer.snowball.ext.norwegianStemmer;
import com.nexwave.stemmer.snowball.ext.portugueseStemmer;
import com.nexwave.stemmer.snowball.ext.romanianStemmer;
import com.nexwave.stemmer.snowball.ext.russianStemmer;
import com.nexwave.stemmer.snowball.ext.spanishStemmer;
import com.nexwave.stemmer.snowball.ext.swedishStemmer;
import com.nexwave.stemmer.snowball.ext.turkishStemmer;

//CJK Tokenizing
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.cjk.CJKTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;


/**
 * Parser for the html files generated by DITA-OT.
 * Extracts the title, the shortdesc and the text within the "content" div tag. <div id="content">
 * NOTE: This indexes only the content under a tag with ID "content".
 * Wrap html content with a div tag with id "content" to index relevant parts of your page.
 *
 * @author N. Quaine
 * @author Kasun Gajasinghe <http://kasunbg.blogspot.com>
 * @version 2.0 2010
 */
public class SaxHTMLIndex extends SaxDocFileParser {

    //KasunBG: apparently tempDico stores all the keywords and a pointer to the files containing the index in a Map
    //example: ("keyword1", "0,2,4"), ("docbook", "1,2,5") 
    private Map<String, String> tempDico;
    private int i = 0;
    private ArrayList<String> cleanUpList = null;
    private ArrayList<String> cleanUpPunctuation = null;

    // START OXYGEN PATCH, scoring for HTML elements
    private int SCORING_FOR_H1 = 50;
    private int SCORING_FOR_H2 = 45;
    private int SCORING_FOR_H3 = 40;
    private int SCORING_FOR_H4 = 35;
    private int SCORING_FOR_H5 = 30;
    private int SCORING_FOR_H6 = 25;
    private int SCORING_FOR_BOLD = 5;
    private int SCORING_FOR_ITALIC = 3;
    private int SCORING_FOR_NORMAL_TEXT = 1;
    private int SCORING_FOR_KEYWORD = 100;
    private int SCORING_FOR_INDEXTERM = 75;

    /**
     * The list with the word and scoring object
     */
    private List<WordAndScoring> wsList = null;

    /**
     * Used for Oxygen TestCases
     *
     * @return the wsList
     */
    public List<WordAndScoring> getWsList() {
        return wsList;
    }
    // END OXYGEN PATCH
    //methods

    /**
     * Constructor
     */
    public SaxHTMLIndex() {
        super();
    }

    /**
     * Constructor
     *
     * @param cleanUpStrings
     */
    public SaxHTMLIndex(ArrayList<String> cleanUpStrings) {
        super();
        cleanUpList = cleanUpStrings;
    }

    /**
     * Constructor
     *
     * @param cleanUpStrings
     * @param cleanUpChars
     */
    public SaxHTMLIndex(ArrayList<String> cleanUpStrings, ArrayList<String> cleanUpChars) {
        super();
        cleanUpList = cleanUpStrings;
        cleanUpPunctuation = cleanUpChars;
    }

    /**
     * Initializer
     *
     * @param tempMap
     */
    public int init(Map<String, String> tempMap) {
        tempDico = tempMap;
        return 0;
    }

    /**
     * Parses the file to extract all the words for indexing and
     * some data characterizing the file.
     *
     * @param file            contains the fullpath of the document to parse
     * @param indexerLanguage this will be used to tell the program which stemmer to be used.
     * @param stem            if true then generate js files with words stemmed
     * @return a DitaFileInfo object filled with data describing the file
     */
    public DocFileInfo runExtractData(File file, String indexerLanguage, boolean stem) {
        //initialization
        fileDesc = new DocFileInfo(file);
        strbf = new StringBuffer("");

        // Fill strbf by parsing the file
        parseDocument(file);

        String str = cleanBuffer(strbf);
        str = str.replaceAll("\\s+", " ");   //there's still redundant spaces in the middle
//		System.out.println(file.toString()+" "+ str +"\n");
        // START OXYGEN PATCH
//		String[] items = str.split("\\s");      //contains all the words in the array
        // END OXYGEN PATCH

        //get items one-by-one, tunnel through the stemmer, and get the stem.
        //Then, add them to tempSet
        //Do Stemming for words in items
        //TODO currently, stemming support is for english and german only. Add support for other languages as well.

        // START OXYGEN PATCH
        wsList = new ArrayList<WordAndScoring>();
        // START OXYGEN PATCH, create the words and scoring list
//        String[] tokenizedItems;
        // END OXYGEN PATCH
        if (indexerLanguage.equalsIgnoreCase("ja") || indexerLanguage.equalsIgnoreCase("zh")
                || indexerLanguage.equalsIgnoreCase("ko")) {
            LinkedList<String> tokens = new LinkedList<String>();
            try {
                //EXM-21501 Oxygen patch, replace the extra "@@@"s.
                str = str.replaceAll("@@@([^\\s]*)@@@", "");
                CJKAnalyzer analyzer = new CJKAnalyzer(org.apache.lucene.util.Version.LUCENE_30);
                Reader reader = new StringReader(str);
                TokenStream stream = analyzer.tokenStream("", reader);
                TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
                OffsetAttribute offAtt = (OffsetAttribute) stream.addAttribute(OffsetAttribute.class);

                while (stream.incrementToken()) {
                    String term = termAtt.term();
                    tokens.add(term);
                    WordAndScoring ws = new WordAndScoring(term, term, 1);
                    boolean found = false;
                    for (WordAndScoring aWsList : wsList) {
                        // If the stem of the current word is already in list,
                        // do not add the word in the list, just recompute scoring
                        if (aWsList.getStem().equals(ws.getStem())) {
                            found = true;
                            int scoring = aWsList.getScoring();
                            aWsList.setScoring(scoring + ws.getScoring());
                            break;
                        }

                    }
                    if (!found) {
                        wsList.add(ws);
                    }
                }
                // START OXYGEN PATCH
                //tokenizedItems = tokens.toArray(new String[tokens.size()]);
                // END OXYGEN PATCH

            } catch (IOException ex) {
                // START OXYGEN PATCH
//                tokenizedItems = items;
                // END OXYGEN PATCH
                System.out.println("Error tokenizing content using CJK Analyzer. IOException");
                ex.printStackTrace();
            }
        } else {
            SnowballStemmer stemmer;
            if (indexerLanguage.equalsIgnoreCase("en")) {
                stemmer = new EnglishStemmer();
            } else if (indexerLanguage.equalsIgnoreCase("de")) {
                stemmer = new GermanStemmer();
            } else if (indexerLanguage.equalsIgnoreCase("fr")) {
                stemmer = new FrenchStemmer();
            } else {
                stemmer = null;//Languages which stemming is not yet supported.So, No stemmers will be used.
            }
            // START OXYGEN PATCH
            wsList = new ArrayList<WordAndScoring>();
            StringTokenizer st = new StringTokenizer(str, " ");
            // Tokenize the string and populate the words and scoring list
            while (st.hasMoreTokens()) {
                String token = st.nextToken();
                WordAndScoring ws = getWordAndScoring(token, stemmer, stem);
                if (ws != null) {
                    boolean found = false;
                    for (WordAndScoring aWsList : wsList) {
                        // If the stem of the current word is already in list,
                        // do not add the word in the list, just recompute scoring
                        if (aWsList.getStem().equals(ws.getStem())) {
                            found = true;
                            int scoring = aWsList.getScoring();
                            aWsList.setScoring(scoring + ws.getScoring());
                            break;
                        }
                    }
                    if (!found) {
                        wsList.add(ws);
                    }
                }
            }
//            if(stemmer != null)             //If a stemmer available
//                tokenizedItems = stemmer.doStem(items.toArray(new String[0]));
//            else                            //if no stemmer available for the particular language
//                tokenizedItems = items.toArray(new String[0]);
            // END OXYGEN PATCH

        }

        /* for(String stemmedItem: tokenizedItems){
            System.out.print(stemmedItem+"| ");
        }*/

        // START OXYGEN PATCH
//		//items: remove the duplicated strings first
//		HashSet <String> tempSet = new HashSet<String>();
//      tempSet.addAll(Arrays.asList(tokenizedItems));
//		Iterator it = tempSet.iterator();
        // Iterate over the words and scoring list
        Iterator<WordAndScoring> it = wsList.iterator();
        WordAndScoring s;
        while (it.hasNext()) {
            s = it.next();
            // Do not add results from 'toc.html'
            if (s != null && tempDico.containsKey(s.getStem())) {
                String temp = tempDico.get(s.getStem());
                temp = temp.concat(",").concat(Integer.toString(i))
                        // Concat also the scoring for the stem
                        .concat("*").concat(Integer.toString(s.getScoring()))
                        ;
                //System.out.println("temp="+s+"="+temp);
                tempDico.put(s.getStem(), temp);
            } else if (s != null) {
                String temp = null;
                temp = Integer.toString(i).concat("*").concat(Integer.toString(s.getScoring()));
                tempDico.put(s.getStem(), temp);
            }
            // END OXYGEN PATCH
        }

        i++;
        return fileDesc;
    }

    // START OXYGEN PATCH

    /**
     * Get the word, stem and scoring for the given token.
     *
     * @param token      The token to parse.
     * @param stemmer    The stemmer.
     * @param doStemming If true then generate js files with words stemmed.
     * @return the word, stem and scoring for the given token.
     */
    private WordAndScoring getWordAndScoring(String token, SnowballStemmer stemmer, boolean doStemming) {
        WordAndScoring wordScoring = null;
        if (token.indexOf("@@@") != -1 && token.indexOf("@@@") != token.lastIndexOf("@@@")) {
            // Extract the word from token
            String word = token.substring(0, token.indexOf("@@@"));
            if (word.length() > 0) {
                // Extract the element name from token
                String elementName = token.substring(token.indexOf("@@@elem_") + "@@@elem_".length(), token.lastIndexOf("@@@"));
                // Compute scoring
                int scoring = SCORING_FOR_NORMAL_TEXT;
                if ("h1".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H1;
                } else if ("h2".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H2;
                } else if ("h3".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H3;
                } else if ("h4".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H4;
                } else if ("h5".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H5;
                } else if ("h6".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_H6;
                } else if ("em".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_ITALIC;
                } else if ("strong".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_BOLD;
                } else if ("meta_keywords".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_KEYWORD;
                } else if ("meta_indexterms".equalsIgnoreCase(elementName)) {
                    scoring = SCORING_FOR_INDEXTERM;
                }
                // Get the stemmed word
                String stemWord = word;
                if (stemmer != null && doStemming) {
                    stemWord = stemmer.doStem(word);
                }
                wordScoring = new WordAndScoring(word, stemWord, scoring);
            }
        } else {
            // The token contains only the word
            String stemWord = token;
            // Stem the word
            if (stemmer != null && doStemming) {
                stemWord = stemmer.doStem(token);
            }
            wordScoring = new WordAndScoring(token, stemWord, SCORING_FOR_NORMAL_TEXT);
        }
        return wordScoring;
    }
    // END OXYGEN PATCH

    /**
     * Cleans the string buffer containing all the text retrieved from
     * the html file:  remove punctuation, clean white spaces, remove the words
     * which you do not want to index.
     * NOTE: You may customize this function:
     * This version takes into account english and japanese. Depending on your
     * needs,
     * you may have to add/remove some characters/words through props files
     * or by modifying tte default code,
     * you may want to separate the language processing (doc only in japanese,
     * doc only in english, check the language metadata ...).
     */
    private String cleanBuffer(StringBuffer strbf) {
        String str = strbf.toString().toLowerCase();
        StringBuffer tempStrBuf = new StringBuffer("");
        StringBuffer tempCharBuf = new StringBuffer("");
        if ((cleanUpList == null) || (cleanUpList.isEmpty())) {
            // Default clean-up

            // Should perhaps eliminate the words at the end of the table?
            tempStrBuf.append("(?i)\\bthe\\b|\\ba\\b|\\ban\\b|\\bto\\b|\\band\\b|\\bor\\b");//(?i) ignores the case
            tempStrBuf.append("|\\bis\\b|\\bare\\b|\\bin\\b|\\bwith\\b|\\bbe\\b|\\bcan\\b");
            tempStrBuf.append("|\\beach\\b|\\bhas\\b|\\bhave\\b|\\bof\\b|\\b\\xA9\\b|\\bnot\\b");
            tempStrBuf.append("|\\bfor\\b|\\bthis\\b|\\bas\\b|\\bit\\b|\\bhe\\b|\\bshe\\b");
            tempStrBuf.append("|\\byou\\b|\\bby\\b|\\bso\\b|\\bon\\b|\\byour\\b|\\bat\\b");
            tempStrBuf.append("|\\b-or-\\b|\\bso\\b|\\bon\\b|\\byour\\b|\\bat\\b");
            tempStrBuf.append("|\\bI\\b|\\bme\\b|\\bmy\\b");

            str = str.replaceFirst("Copyright � 1998-2007 NexWave Solutions.", " ");


            //nqu 25.01.2008 str = str.replaceAll("\\b.\\b|\\\\", " ");
            // remove contiguous white charaters
            //nqu 25.01.2008 str = str.replaceAll("\\s+", " ");
        } else {
            // Clean-up using the props files
            tempStrBuf.append("\\ba\\b");
            for (String aCleanUp : cleanUpList) {
                tempStrBuf.append("|\\b").append(aCleanUp).append("\\b");
            }
        }
        if ((cleanUpPunctuation != null) && (!cleanUpPunctuation.isEmpty())) {
            tempCharBuf.append("\\u3002");
            for (String aCleanUpPunctuation : cleanUpPunctuation) {
                tempCharBuf.append("|").append(aCleanUpPunctuation);
            }
        }

        str = minimalClean(str, tempStrBuf, tempCharBuf);
        return str;
    }

    // OXYGEN PATCH, moved method in superclass
//	private String minimalClean(String str, StringBuffer tempStrBuf, StringBuffer tempCharBuf) {
//		String tempPunctuation = new String(tempCharBuf);
//
//		str = str.replaceAll("\\s+", " ");
//		str = str.replaceAll("->", " ");
//		str = str.replaceAll(IndexerConstants.EUPUNCTUATION1, " ");
//		str = str.replaceAll(IndexerConstants.EUPUNCTUATION2, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION1, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION2, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION3, " ");
//		if (tempPunctuation.length() > 0)
//		{
//			str = str.replaceAll(tempPunctuation, " ");
//		}
//
//		//remove useless words
//		str = str.replaceAll(tempStrBuf.toString(), " ");
//
//		// Redo punctuation after removing some words: (TODO: useful?)
//		str = str.replaceAll(IndexerConstants.EUPUNCTUATION1, " ");
//		str = str.replaceAll(IndexerConstants.EUPUNCTUATION2, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION1, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION2, " ");
//		str = str.replaceAll(IndexerConstants.JPPUNCTUATION3, " ");
//		if (tempPunctuation.length() > 0)
//		{
//			str = str.replaceAll(tempPunctuation, " ");
//		}		return str;
//	}
    // END OXYGEN PATCH

}
