[metacat-cvs] metacat/src/edu/ucsb/nceas/metacat IndexingQueue.java IndexingTimerTask.java
Saurabh Garg
sgarg at ecoinformatics.org
Thu Nov 10 14:13:04 PST 2005
sgarg 05/11/10 14:13:04
Added: src/edu/ucsb/nceas/metacat IndexingQueue.java
IndexingTimerTask.java
Log:
New code for run an indexing queue and indexing thread which runs every 24 hours
Revision Changes Path
1.1 metacat/src/edu/ucsb/nceas/metacat/IndexingQueue.java
Index: IndexingQueue.java
===================================================================
/**
* '$RCSfile: IndexingQueue.java,v $'
* Purpose: A Class that tracks sessions for MetaCatServlet users.
* Copyright: 2000 Regents of the University of California and the
* National Center for Ecological Analysis and Synthesis
* Authors: Matt Jones
* Release: @release@
*
* '$Author: sgarg $'
* '$Date: 2005/11/10 22:13:04 $'
* '$Revision: 1.1 $'
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package edu.ucsb.nceas.metacat;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Vector;
import edu.ucsb.nceas.metacat.MetaCatUtil;
import org.apache.log4j.Logger;
public class IndexingQueue {
private Logger logMetacat = Logger.getLogger(IndexingQueue.class);
private Vector indexingQueue = new Vector();
private Vector currentThreads = new Vector();
private static IndexingQueue instance = null;
final static int NUMBEROFINDEXINGTHREADS =
Integer.parseInt(MetaCatUtil.getOption("numberOfIndexingThreads"));
private int sleepTime = 2000;
public IndexingQueue() {
for (int i = 0; i < NUMBEROFINDEXINGTHREADS; i++) {
Thread thread = new IndexingTask();
thread.start();
}
}
public static synchronized IndexingQueue getInstance(){
if (instance == null) {
instance = new IndexingQueue();
}
return instance;
}//getInstance
public void add(String docid) {
synchronized (indexingQueue) {
indexingQueue.add(new IndexingQueueObject(0, docid));
indexingQueue.notify();
}
}
public void add(IndexingQueueObject queueObject) {
synchronized (indexingQueue) {
indexingQueue.add(queueObject);
indexingQueue.notify();
}
}
protected IndexingQueueObject getNext() {
IndexingQueueObject returnVal = null;
synchronized (indexingQueue) {
while (indexingQueue.isEmpty()) {
try {
indexingQueue.wait();
} catch (InterruptedException ex) {
System.err.println("Interrupted");
}
}
returnVal = (IndexingQueueObject) indexingQueue.get(0);
indexingQueue.remove(0);
}
return returnVal;
}
}
class IndexingTask extends Thread {
private Logger logMetacat = Logger.getLogger(IndexingTask.class);
protected final long INDEXDELAY = 5000;
public void run() {
while (true) {
// blocks until job
IndexingQueueObject returnVal =
IndexingQueue.getInstance().getNext();
String docid = returnVal.getDocid();
try {
checkDocumentTable(docid);
DocumentImpl doc = new DocumentImpl(docid, false);
logMetacat.warn("Calling buildIndex for " + docid);
doc.buildIndex();
} catch (Exception e) {
logMetacat.warn("Exception: " + e);
e.printStackTrace();
if(returnVal.getCount() < 25){
returnVal.setCount(returnVal.getCount()+1);
// add the docid back to the list
IndexingQueue.getInstance().add(docid);
} else {
logMetacat.fatal("Docid " + returnVal.getDocid()
+ " has been inserted to IndexingQueue "
+ "more than 25 times.");
}
}
}
}
private void checkDocumentTable(String docid) throws Exception{
DBConnection dbConn = null;
int serialNumber = -1;
String revision = docid.substring(docid.lastIndexOf(".")+1,docid.length());
docid = docid.substring(0,docid.lastIndexOf("."));
logMetacat.warn("docid is " + docid
+ " and revision is " + revision);
try {
// Opening separate db connection for writing XML Index
dbConn = DBConnectionPool
.getDBConnection("DBSAXHandler.checkDocumentTable");
serialNumber = dbConn.getCheckOutSerialNumber();
// the following while loop construct checks to make sure that
// the docid of the document that we are trying to index is already
// in the xml_documents table. if this is not the case, the foreign
// key relationship between xml_documents and xml_index is
// temporarily broken causing multiple problems.
boolean inxmldoc = false;
long startTime = System.currentTimeMillis();
while (!inxmldoc) {
String xmlDocumentsCheck = "select distinct docid from xml_documents"
+ " where docid ='"
+ docid
+ "' and "
+ " rev ='"
+ revision + "'";
PreparedStatement xmlDocCheck = dbConn
.prepareStatement(xmlDocumentsCheck);
// Increase usage count
dbConn.increaseUsageCount(1);
xmlDocCheck.execute();
ResultSet doccheckRS = xmlDocCheck.getResultSet();
boolean tableHasRows = doccheckRS.next();
if (tableHasRows) {
inxmldoc = true;
}
doccheckRS.close();
xmlDocCheck.close();
// make sure the while loop will be ended in reseaonable time
long stopTime = System.currentTimeMillis();
if ((stopTime - startTime) > INDEXDELAY) {
logMetacat.warn("Couldn't find the docid:" + docid + " for indexing in "
+ "reseaonable time!");
throw new Exception(
"Couldn't find the docid for index build in "
+ "reseaonable time!");
}
}//while
} catch (SQLException e) {
e.printStackTrace();
} finally {
DBConnectionPool.returnDBConnection(dbConn, serialNumber);
}//finally
}
}
class IndexingQueueObject{
// the docid of the document to be indexed.
private String docid;
// the count of number of times the document has been in the queue
private int count;
IndexingQueueObject(int count, String docid){
this.count = count;
this.docid = docid;
}
public int getCount(){
return count;
}
public String getDocid(){
return docid;
}
public void setCount(int count){
this.count = count;
}
public void setDocid(String docid){
this.docid = docid;
}
}
1.1 metacat/src/edu/ucsb/nceas/metacat/IndexingTimerTask.java
Index: IndexingTimerTask.java
===================================================================
/**
* '$RCSfile: IndexingTimerTask.java,v $'
* Purpose: A Class that tracks sessions for MetaCatServlet users.
* Copyright: 2000 Regents of the University of California and the
* National Center for Ecological Analysis and Synthesis
* Authors: Matt Jones
* Release: @release@
*
* '$Author: sgarg $'
* '$Date: 2005/11/10 22:13:04 $'
* '$Revision: 1.1 $'
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package edu.ucsb.nceas.metacat;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.TimerTask;
import edu.ucsb.nceas.metacat.MetaCatUtil;
import org.apache.log4j.Logger;
public class IndexingTimerTask extends TimerTask{
private Logger logMetacat = Logger.getLogger(IndexingTimerTask.class);
int count = 0;
/*
* Run a separate thread to build the XML index for this document. This
* thread is run asynchronously in order to more quickly return control to
* the submitting user. The run method checks to see if the document has
* been fully inserted before trying to update the xml_index table.
*/
public void run()
{
DBConnection dbConn = null;
int serialNumber = 0;
try{
logMetacat.warn("Running indexing timer task");
dbConn = DBConnectionPool.getDBConnection("IndexingThread.run");
serialNumber = dbConn.getCheckOutSerialNumber();
String xmlDocumentsCheck =
MetaCatUtil.dbAdapter.getLeftJoinQuery("a.docid, a.rev", "xml_documents",
"xml_index", "a.docid = b.docid", "b.docid is NULL AND "
+ "(a.doctype like 'eml://ecoinformatics.org/eml-2.0.0' "
+ "or a.doctype like 'eml://ecoinformatics.org/eml-2.0.1')");
PreparedStatement xmlDocCheck = dbConn
.prepareStatement(xmlDocumentsCheck);
// Increase usage count
dbConn.increaseUsageCount(1);
xmlDocCheck.execute();
ResultSet rs = xmlDocCheck.getResultSet();
boolean tableHasRows = rs.next();
while (tableHasRows) {
String docid = rs.getString(1);
String rev = rs.getString(2);
IndexingQueue.getInstance().add(docid + "." + rev);
tableHasRows = rs.next();
}
rs.close();
xmlDocCheck.close();
} catch (SQLException se){
se.printStackTrace();
} catch (Exception e){
e.printStackTrace();
}finally {
DBConnectionPool.returnDBConnection(dbConn, serialNumber);
}
logMetacat.warn("Indexing timer task returning");
count++;
}
}
More information about the Metacat-cvs
mailing list