/**

  * Distribution License:

  * JSword is free software; you can redistribute it and/or modify it under

  * the terms of the GNU Lesser General Public License, version 2.1 or later

  * as published by the Free Software Foundation. This program is distributed

  * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even

  * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

  * See the GNU Lesser General Public License for more details.

  *

  * The License is available on the internet at:

  *      http://www.gnu.org/copyleft/lgpl.html

  * or by writing to:

  *      Free Software Foundation, Inc.

  *      59 Temple Place - Suite 330

  *      Boston, MA 02111-1307, USA

  *

  * © CrossWire Bible Society, 2005 - 2016

  *

  */

 package org.crosswire.jsword.index.lucene;

 import java.io.Closeable;

 import java.io.File;

 import java.io.IOException;

 import java.net.URI;

 import java.util.ArrayList;

 import java.util.List;

 import org.apache.lucene.analysis.Analyzer;

 import org.apache.lucene.document.Document;

 import org.apache.lucene.document.Field;

 import org.apache.lucene.index.IndexWriter;

 import org.apache.lucene.queryParser.ParseException;

 import org.apache.lucene.queryParser.QueryParser;

 import org.apache.lucene.search.IndexSearcher;

 import org.apache.lucene.search.Query;

 import org.apache.lucene.search.ScoreDoc;

 import org.apache.lucene.search.Searcher;

 import org.apache.lucene.search.TopScoreDocCollector;

 import org.apache.lucene.store.Directory;

 import org.apache.lucene.store.FSDirectory;

 import org.apache.lucene.util.Version;

 import org.crosswire.common.progress.JobManager;

 import org.crosswire.common.progress.Progress;

 import org.crosswire.common.util.FileUtil;

 import org.crosswire.common.util.IOUtil;

 import org.crosswire.common.util.NetUtil;

 import org.crosswire.common.util.Reporter;

 import org.crosswire.jsword.JSMsg;

 import org.crosswire.jsword.book.Book;

 import org.crosswire.jsword.book.BookData;

 import org.crosswire.jsword.book.BookException;

 import org.crosswire.jsword.book.FeatureType;

 import org.crosswire.jsword.book.OSISUtil;

 import org.crosswire.jsword.index.AbstractIndex;

 import org.crosswire.jsword.index.IndexPolicy;

 import org.crosswire.jsword.index.IndexStatus;

 import org.crosswire.jsword.index.lucene.analysis.LuceneAnalyzer;

 import org.crosswire.jsword.index.search.SearchModifier;

 import org.crosswire.jsword.passage.AbstractPassage;

 import org.crosswire.jsword.passage.Key;

 import org.crosswire.jsword.passage.NoSuchKeyException;

 import org.crosswire.jsword.passage.NoSuchVerseException;

 import org.crosswire.jsword.passage.PassageTally;

 import org.crosswire.jsword.passage.Verse;

 import org.crosswire.jsword.passage.VerseFactory;

 import org.crosswire.jsword.versification.Versification;

 import org.crosswire.jsword.versification.system.Versifications;

 import org.jdom2.Element;

 import org.slf4j.Logger;

 import org.slf4j.LoggerFactory;

 /**

  * Implement the SearchEngine using Lucene as the search engine.

  * 

  * @see gnu.lgpl.License The GNU Lesser General Public License for details.

  * @author Joe Walker

  */

 public class LuceneIndex extends AbstractIndex implements Closeable {

     /*

      * The following fields are named the same as Sword in the hopes of sharing

      * indexes.

      */

     /**

      * The Lucene field for the osisID

      */

     public static final String FIELD_KEY = "key";

     /**

      * The Lucene field for the text contents

      */

     public static final String FIELD_BODY = "content";

     /**

      * The Lucene field for the strong numbers

      */

     public static final String FIELD_STRONG = "strong";

     /**

      * The Lucene field for headings

      */

     public static final String FIELD_HEADING = "heading";

     /**

      * The Lucene field for cross references

      */

     public static final String FIELD_XREF = "xref";

     /**

      * The Lucene field for the notes

      */

     public static final String FIELD_NOTE = "note";

     /**

      * Combines the strong numbers with the morphology field

      */

     public static final String FIELD_MORPHOLOGY = "morph";

     /**

      * Combines the strong numbers with the morphology field

      */

     public static final String FIELD_INTRO = "intro";

     /**

      * An estimate of the percent of time spent indexing.

      * The remaining time, if any, is spent doing cleanup.

      */

     private static final int WORK_ESTIMATE = 98;

     /**

      * Read an existing index and use it.

      * 

      * @param book the book

      * @param storage 

      * @throws BookException

      *             If we fail to read the index files

      */

     public LuceneIndex(Book book, URI storage) throws BookException {

         this.book = book;

         try {

             this.path = NetUtil.getAsFile(storage).getCanonicalPath();

         } catch (IOException ex) {

             // TRANSLATOR: Error condition: Could not initialize a search index.

             throw new BookException(JSMsg.gettext("Failed to initialize Lucene search engine."), ex);

         }

         initDirectoryAndSearcher();

     }

     /**

      * Generate an index to use, telling the job about progress as you go.

      * 

      * @param book the book

      * @param storage 

      * @param policy 

      * @throws BookException

      *             If we fail to read the index files

      */

     public LuceneIndex(Book book, URI storage, IndexPolicy policy) throws BookException {

         this.book = book;

         File finalPath = null;

         try {

             finalPath = NetUtil.getAsFile(storage);

             this.path = finalPath.getCanonicalPath();

         } catch (IOException ex) {

             // TRANSLATOR: Error condition: Could not initialize a search index. Lucene is the name of the search technology being used.

             throw new BookException(JSMsg.gettext("Failed to initialize Lucene search engine."), ex);

         }

         // TRANSLATOR: Progress label indicating the start of indexing. {0} is a placeholder for the book's short name.

         String jobName = JSMsg.gettext("Creating index. Processing {0}", book.getInitials());

         Progress job = JobManager.createJob(String.format(Progress.CREATE_INDEX, book.getInitials()), jobName, Thread.currentThread());

         job.beginJob(jobName);

         IndexStatus finalStatus = IndexStatus.UNDONE;

         List<Key> errors = new ArrayList<Key>();

         // Build to another location and rename in the end.

         File tempPath = new File(path + '.' + IndexStatus.CREATING.toString());

         // Ensure that the temp path is gone

         // It is not good for it to have been leftover from before.

         if (tempPath.exists()) {

             FileUtil.delete(tempPath);

         }

         try {

             // When misconfigured, this can throw errors.

             Analyzer analyzer = new LuceneAnalyzer(book);

             // Lock on metadata to allow creation of multiple indexes, so long as they are on different books.

             // Otherwise lock on a single object to make this serial

             Object mutex = policy.isSerial() ? CREATING : book.getBookMetaData();

             synchronized (mutex) {

                 book.setIndexStatus(IndexStatus.CREATING);

                 IndexWriter writer = null;

                 try {

                     // Write the core index to disk.

                     final Directory destination = FSDirectory.open(new File(tempPath.getCanonicalPath()));

                     writer = new IndexWriter(destination, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);

                     writer.setRAMBufferSizeMB(policy.getRAMBufferSize());

                     generateSearchIndexImpl(job, errors, writer, book.getGlobalKeyList(), 0, policy);

                 } finally {

                     if (writer != null) {

                         writer.close();

                     }

                 }

                 job.setCancelable(false);

                 if (!job.isFinished()) {

                     if (!tempPath.renameTo(finalPath)) {

                         // TRANSLATOR: The search index could not be moved to it's final location.

                         throw new BookException(JSMsg.gettext("Installation failed."));

                     }

                 }

                 if (finalPath.exists()) {

                     finalStatus = IndexStatus.DONE;

                 }

                 if (!errors.isEmpty()) {

                     StringBuilder buf = new StringBuilder();

                     for (Key error : errors) {

                         buf.append(error);

                         buf.append('\n');

                     }

                     // TRANSLATOR: It is likely that one or more verses could not be indexed due to errors in those verses.

                     // This message gives a listing of them to the user.

                     Reporter.informUser(this, JSMsg.gettext("The following verses have errors and could not be indexed\n{0}", buf));

                 }

                 initDirectoryAndSearcher();

             }

         } catch (IOException ex) {

             job.cancel();

             // TRANSLATOR: Common error condition: Some error happened while creating a search index.

             throw new BookException(JSMsg.gettext("Failed to initialize Lucene search engine."), ex);

         } finally {

             book.setIndexStatus(finalStatus);

             job.done();

             // Ensure that the temp path is gone - errors can leave it there and cause further problems.

             if (tempPath.exists()) {

                 FileUtil.delete(tempPath);

             }

         }

     }

     /**

      * Initializes the directory and searcher.

      */

     private void initDirectoryAndSearcher() {

         try {

             directory = FSDirectory.open(new File(path));

             searcher = new IndexSearcher(directory, true);

         } catch (IOException ex) {

             log.warn("second load failure", ex);

         }

     }

     /* (non-Javadoc)

      * @see org.crosswire.jsword.index.Index#find(java.lang.String)

      */

     public Key find(String search) throws BookException {

         String v11nName = book.getBookMetaData().getProperty("Versification").toString();

         Versification v11n = Versifications.instance().getVersification(v11nName);

         SearchModifier modifier = getSearchModifier();

         Key results = null;

         if (search != null) {

             Throwable theCause = null;

             try {

                 Analyzer analyzer = new LuceneAnalyzer(book);

                 QueryParser parser = new QueryParser(Version.LUCENE_29, LuceneIndex.FIELD_BODY, analyzer);

                 parser.setAllowLeadingWildcard(true);

                 Query query = parser.parse(search);

                 log.info("ParsedQuery- {}", query.toString());

                 // For ranking we use a PassageTally

                 if (modifier != null && modifier.isRanked()) {

                     PassageTally tally = new PassageTally(v11n);

                     tally.raiseEventSuppresion();

                     tally.raiseNormalizeProtection();

                     results = tally;

                     TopScoreDocCollector collector = TopScoreDocCollector.create(modifier.getMaxResults(), false);

                     searcher.search(query, collector);

                     tally.setTotal(collector.getTotalHits());

                     ScoreDoc[] hits = collector.topDocs().scoreDocs;

                     for (int i = 0; i < hits.length; i++) {

                         int docId = hits[i].doc;

                         Document doc = searcher.doc(docId);

                         Key key = VerseFactory.fromString(v11n, doc.get(LuceneIndex.FIELD_KEY));

                         // PassageTally understands a score of 0 as the verse

                         // not participating

                         int score = (int) (hits[i].score * 100 + 1);

                         tally.add(key, score);

                     }

                     tally.lowerNormalizeProtection();

                     tally.lowerEventSuppressionAndTest();

                 } else {

                     results = book.createEmptyKeyList();

                     // If we have an abstract passage,

                     // make sure it does not try to fire change events.

                     AbstractPassage passage = null;

                     if (results instanceof AbstractPassage) {

                         passage = (AbstractPassage) results;

                         passage.raiseEventSuppresion();

                         passage.raiseNormalizeProtection();

                     }

                     searcher.search(query, new VerseCollector(v11n, searcher, results));

                     if (passage != null) {

                         passage.lowerNormalizeProtection();

                         passage.lowerEventSuppressionAndTest();

                     }

                 }

             } catch (IOException e) {

                 // The VerseCollector may throw IOExceptions that merely wrap a NoSuchVerseException

                 Throwable cause = e.getCause();

                 theCause = cause instanceof NoSuchVerseException ? cause : e;

             } catch (NoSuchVerseException e) {

                 theCause = e;

             } catch (ParseException e) {

                 theCause = e;

             }

             if (theCause != null) {

                 // TRANSLATOR: Error condition: An unexpected error happened that caused search to fail.

                 throw new BookException(JSMsg.gettext("Search failed."), theCause);

             }

         }

         if (results == null) {

             if (modifier != null && modifier.isRanked()) {

                 results = new PassageTally(v11n);

             } else {

                 results = book.createEmptyKeyList();

             }

         }

         return results;

     }

     /* (non-Javadoc)

      * @see org.crosswire.jsword.index.Index#getKey(java.lang.String)

      */

     public Key getKey(String name) throws NoSuchKeyException {

         return book.getKey(name);

     }

     /* (non-Javadoc)

      * @see org.crosswire.jsword.index.Index#close()

      */

     public final void close() {

         IOUtil.close(searcher);

         searcher = null;

         IOUtil.close(directory);

         directory = null;

     }

     /**

      * Dig down into a Key indexing as we go.

      * @param policy 

      */

     private void generateSearchIndexImpl(Progress job, List<Key> errors, IndexWriter writer, Key key, int count, IndexPolicy policy) throws BookException, IOException {

         String v11nName = null;

         if (book.getBookMetaData().getProperty("Versification") != null) {

             v11nName = book.getBookMetaData().getProperty("Versification").toString();

         }

         Versification v11n = Versifications.instance().getVersification(v11nName);

         boolean includeStrongs = book.getBookMetaData().hasFeature(FeatureType.STRONGS_NUMBERS) && policy.isStrongsIndexed();

         boolean includeXrefs = book.getBookMetaData().hasFeature(FeatureType.SCRIPTURE_REFERENCES) && policy.isXrefIndexed();

         boolean includeNotes = book.getBookMetaData().hasFeature(FeatureType.FOOTNOTES) && policy.isNoteIndexed();

         boolean includeHeadings = book.getBookMetaData().hasFeature(FeatureType.HEADINGS) && policy.isTitleIndexed();

         boolean includeMorphology = book.getBookMetaData().hasFeature(FeatureType.MORPHOLOGY) && policy.isMorphIndexed();

         String oldRootName = "";

         int percent = 0;

         String rootName = "";

         BookData data = null;

         Element osis = null;

         // Set up for reuse.

         Document doc = new Document();

         Field keyField = new Field(FIELD_KEY, "", Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO);

         Field bodyField = new Field(FIELD_BODY, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         Field introField = new Field(FIELD_INTRO, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         Field strongField = new Field(FIELD_STRONG, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES);

         Field xrefField = new Field(FIELD_XREF, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         Field noteField = new Field(FIELD_NOTE, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         Field headingField = new Field(FIELD_HEADING, "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         Field morphologyField  = new Field(FIELD_MORPHOLOGY , "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);

         int size = key.getCardinality();

         int subCount = count;

         log.debug("Number of keys = {}", Integer.toString(size));

         for (Key subkey : key) {

             // Bibles and verse based commentaries don't have keys with children.

             // However, tree keyed Books do. So we only index the leaf keys.

             // FIXME(DMS): Should not use recursion!!!!

             if (subkey.canHaveChildren()) {

                 generateSearchIndexImpl(job, errors, writer, subkey, subCount, policy);

                 continue;

             }

             data = new BookData(book, subkey);

             osis = null;

             try {

                 osis = data.getOsisFragment(false);

             } catch (BookException e) {

                 errors.add(subkey);

                 continue;

             }

             // Remove all fields from the document

             doc.getFields().clear();

             // Do the actual indexing

             // Always add the key

             keyField.setValue(subkey.getOsisRef());

             doc.add(keyField);

             if (subkey instanceof Verse && ((Verse) subkey).getVerse() == 0) {

                 addField(doc, introField, OSISUtil.getCanonicalText(osis));

             } else {

                 addField(doc, bodyField, OSISUtil.getCanonicalText(osis));

             }

             if (includeStrongs) {

                 addField(doc, strongField, OSISUtil.getStrongsNumbers(osis));

             }

             if (includeXrefs) {

                 // We pass book and key because the xref may not be valid and it needs to be reported.

                 addField(doc, xrefField, OSISUtil.getReferences(this.book, subkey, v11n, osis));

             }

             if (includeNotes) {

                 addField(doc, noteField, OSISUtil.getNotes(osis));

             }

             if (includeHeadings) {

                 String heading = OSISUtil.getHeadings(osis);

                 addField(doc, headingField, heading);

             }

             if (includeMorphology) {

                 addField(doc, morphologyField, OSISUtil.getMorphologiesWithStrong(osis));

             }

             // Add the document if we added more than just the key.

             if (doc.getFields().size() > 1) {

                 writer.addDocument(doc);

             }

             // report progress

             rootName = subkey.getRootName();

             if (!rootName.equals(oldRootName)) {

                 oldRootName = rootName;

                 // Note, this does not cause progress to be updated

                 // It will show up the next time progress is updated.

                 job.setSectionName(rootName);

             }

             subCount++;

             int oldPercent = percent;

             percent = WORK_ESTIMATE * subCount / size;

             // Only send out a max of 95 progress updates

             if (oldPercent != percent) {

                 job.setWork(percent);

             }

             // This could take a long time ...

             Thread.yield();

             if (Thread.currentThread().isInterrupted()) {

                 break;

             }

         }

     }

     /**

      * Add the text to the Field and put the Field in the document,

      * ignoring null and empty text.

      * 

      * @param doc The Document to which the Field should be added

      * @param field The Field to add

      * @param text The text for the field

      */

     private void addField(Document doc, Field field, String text) {

         if (text != null && text.length() > 0) {

             field.setValue(text);

             doc.add(field);

         }

     }

     /**

      * Could be null if the index has been closed down. This is helpful to third party applications which wish to have greater control over 

      * the underlying Lucene functionality.

      * 

      * Note: by using this method, you need to ensure you don't close the searcher while it is being used.

      * See {@link org.crosswire.jsword.index.IndexManager#closeAllIndexes()} for more information

      * @return the searcher

      */

     public Searcher getSearcher() {

         return searcher;

     }

     /**

      * The Book that we are indexing

      */

     private Book book;

     /**

      * The location of this index

      */

     private String path;

     /**

      * The Lucene directory for the path.

      */

     private Directory directory;

     /**

      * The Lucene search engine

      */

     private Searcher searcher;

     /**

      * A synchronization lock point to prevent us from doing 2 index runs at a

      * time.

      */

     private static final Object CREATING = new Object();

     /**

      * The log stream

      */

     private static final Logger log = LoggerFactory.getLogger(LuceneIndex.class);

 }