[metacat-cvs] metacat/src/edu/ucsb/nceas/metacat DocumentImpl.java
Saurabh Garg
sgarg at ecoinformatics.org
Fri Nov 18 13:25:21 PST 2005
sgarg 05/11/18 13:25:21
Modified: src/edu/ucsb/nceas/metacat DocumentImpl.java
Log:
1. Modified buildIndex() so that only one connection is used for both indexing nodes and indexing paths.
2. Modified the indexing algo so that indexing of paths is done while nodes are being indexed. Results in a much faster indexing algo
3. Replaced access to NodeRecord.<variable> with NodeRecord.<getVariable>
Revision Changes Path
1.177 +155 -186 metacat/src/edu/ucsb/nceas/metacat/DocumentImpl.java
Index: DocumentImpl.java
===================================================================
RCS file: /cvs/metacat/src/edu/ucsb/nceas/metacat/DocumentImpl.java,v
retrieving revision 1.176
retrieving revision 1.177
diff -u -r1.176 -r1.177
--- DocumentImpl.java 16 Nov 2005 21:07:52 -0000 1.176
+++ DocumentImpl.java 18 Nov 2005 21:25:21 -0000 1.177
@@ -6,9 +6,9 @@
* Authors: Matt Jones
* Release: @release@
*
- * '$Author: tao $'
- * '$Date: 2005/11/16 21:07:52 $'
- * '$Revision: 1.176 $'
+ * '$Author: sgarg $'
+ * '$Date: 2005/11/18 21:25:21 $'
+ * '$Revision: 1.177 $'
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -1006,10 +1006,10 @@
while (it.hasNext()) {
NodeRecord currentNode = (NodeRecord) it.next();
- logMetacat.info("[Got Node ID: " + currentNode.nodeid + " ("
- + currentNode.parentnodeid + ", " + currentNode.nodeindex
- + ", " + currentNode.nodetype + ", " + currentNode.nodename
- + ", " + currentNode.nodedata + ")]");
+ logMetacat.info("[Got Node ID: " + currentNode.getNodeId() + " ("
+ + currentNode.getParentNodeId() + ", " + currentNode.getNodeIndex()
+ + ", " + currentNode.getNodeType() + ", " + currentNode.getNodeName()
+ + ", " + currentNode.getNodeData() + ")]");
// Print the end tag for the previous node if needed
//
// This is determined by inspecting the parent nodeid for the
@@ -1028,20 +1028,20 @@
// is handled by the NodeComparator class used by the TreeSet
if (!atRootElement) {
NodeRecord currentElement = (NodeRecord) openElements.peek();
- if (currentNode.parentnodeid != currentElement.nodeid) {
- while (currentNode.parentnodeid != currentElement.nodeid) {
+ if (currentNode.getParentNodeId() != currentElement.getNodeId()) {
+ while (currentNode.getParentNodeId() != currentElement.getNodeId()) {
currentElement = (NodeRecord) openElements.pop();
logMetacat.info("\n POPPED: "
- + currentElement.nodename);
+ + currentElement.getNodeName());
if (previousNodeWasElement) {
out.print(">");
previousNodeWasElement = false;
}
- if (currentElement.nodeprefix != null) {
- out.print("</" + currentElement.nodeprefix + ":"
- + currentElement.nodename + ">");
+ if (currentElement.getNodePrefix() != null) {
+ out.print("</" + currentElement.getNodePrefix() + ":"
+ + currentElement.getNodeName() + ">");
} else {
- out.print("</" + currentElement.nodename + ">");
+ out.print("</" + currentElement.getNodeName() + ">");
}
currentElement = (NodeRecord) openElements.peek();
}
@@ -1049,11 +1049,11 @@
}
// Handle the DOCUMENT node
- if (currentNode.nodetype.equals("DOCUMENT")) {
+ if (currentNode.getNodeType().equals("DOCUMENT")) {
out.print("<?xml version=\"1.0\"?>");
// Handle the ELEMENT nodes
- } else if (currentNode.nodetype.equals("ELEMENT")) {
+ } else if (currentNode.getNodeType().equals("ELEMENT")) {
if (atRootElement) {
atRootElement = false;
} else {
@@ -1079,51 +1079,51 @@
}
firstElement = false;
openElements.push(currentNode);
- logMetacat.info("\n PUSHED: " + currentNode.nodename);
+ logMetacat.info("\n PUSHED: " + currentNode.getNodeName());
previousNodeWasElement = true;
- if (currentNode.nodeprefix != null) {
- out.print("<" + currentNode.nodeprefix + ":"
- + currentNode.nodename);
+ if (currentNode.getNodePrefix() != null) {
+ out.print("<" + currentNode.getNodePrefix() + ":"
+ + currentNode.getNodeName());
} else {
- out.print("<" + currentNode.nodename);
+ out.print("<" + currentNode.getNodeName());
}
// if currentNode is inline and handle eml2, set flag proccess
// in
- if (currentNode.nodename != null
- && currentNode.nodename.equals(Eml200SAXHandler.INLINE)
+ if (currentNode.getNodeName() != null
+ && currentNode.getNodeName().equals(Eml200SAXHandler.INLINE)
&& proccessEml2) {
prcocessInlineData = true;
}
// Handle the ATTRIBUTE nodes
- } else if (currentNode.nodetype.equals("ATTRIBUTE")) {
- if (currentNode.nodeprefix != null) {
- out.print(" " + currentNode.nodeprefix + ":"
- + currentNode.nodename + "=\""
- + currentNode.nodedata + "\"");
+ } else if (currentNode.getNodeType().equals("ATTRIBUTE")) {
+ if (currentNode.getNodePrefix() != null) {
+ out.print(" " + currentNode.getNodePrefix() + ":"
+ + currentNode.getNodeName() + "=\""
+ + currentNode.getNodeData() + "\"");
} else {
- out.print(" " + currentNode.nodename + "=\""
- + currentNode.nodedata + "\"");
+ out.print(" " + currentNode.getNodeName() + "=\""
+ + currentNode.getNodeData() + "\"");
}
// Handle the NAMESPACE nodes
- } else if (currentNode.nodetype.equals("NAMESPACE")) {
- out.print(" xmlns:" + currentNode.nodename + "=\""
- + currentNode.nodedata + "\"");
+ } else if (currentNode.getNodeType().equals("NAMESPACE")) {
+ out.print(" xmlns:" + currentNode.getNodeName() + "=\""
+ + currentNode.getNodeData() + "\"");
// Handle the TEXT nodes
- } else if (currentNode.nodetype.equals("TEXT")) {
+ } else if (currentNode.getNodeType().equals("TEXT")) {
if (previousNodeWasElement) {
out.print(">");
}
if (!prcocessInlineData) {
// if it is not inline data just out put data
- out.print(currentNode.nodedata);
+ out.print(currentNode.getNodeData());
} else {
// if it is inline data first to get the inline data
// internal id
- String fileName = currentNode.nodedata;
+ String fileName = currentNode.getNodeData();
String accessfileName = MetaCatUtil
.getDocIdWithoutRevFromInlineDataID(fileName);
// check if user has read permision for this inline data
@@ -1176,23 +1176,23 @@
}// in inlinedata part
previousNodeWasElement = false;
// Handle the COMMENT nodes
- } else if (currentNode.nodetype.equals("COMMENT")) {
+ } else if (currentNode.getNodeType().equals("COMMENT")) {
if (previousNodeWasElement) {
out.print(">");
}
- out.print("<!--" + currentNode.nodedata + "-->");
+ out.print("<!--" + currentNode.getNodeData() + "-->");
previousNodeWasElement = false;
// Handle the PI nodes
- } else if (currentNode.nodetype.equals("PI")) {
+ } else if (currentNode.getNodeType().equals("PI")) {
if (previousNodeWasElement) {
out.print(">");
}
- out.print("<?" + currentNode.nodename + " "
- + currentNode.nodedata + "?>");
+ out.print("<?" + currentNode.getNodeName() + " "
+ + currentNode.getNodeData() + "?>");
previousNodeWasElement = false;
// Handle the DTD nodes (docname, publicid, systemid)
- } else if (currentNode.nodetype.equals(DTD)) {
+ } else if (currentNode.getNodeType().equals(DTD)) {
storedDTD = true;
if (currentNode.getNodeName().equals(DOCNAME)) {
dbDocName = currentNode.getNodeData();
@@ -1233,12 +1233,12 @@
// Print the final end tag for the root element
while (!openElements.empty()) {
NodeRecord currentElement = (NodeRecord) openElements.pop();
- logMetacat.info("\n POPPED: " + currentElement.nodename);
- if (currentElement.nodeprefix != null) {
- out.print("</" + currentElement.nodeprefix + ":"
- + currentElement.nodename + ">");
+ logMetacat.info("\n POPPED: " + currentElement.getNodeName());
+ if (currentElement.getNodePrefix() != null) {
+ out.print("</" + currentElement.getNodePrefix() + ":"
+ + currentElement.getNodeName() + ">");
} else {
- out.print("</" + currentElement.nodename + ">");
+ out.print("</" + currentElement.getNodeName() + ">");
}
}
out.flush();
@@ -1255,6 +1255,7 @@
*/
public void buildIndex() throws McdbException
{
+ logMetacat.warn("buildIndex called on docid " + docid);
TreeSet nodeRecordLists = getNodeRecordList(rootnodeid);
boolean atRootElement = true;
long rootNodeId = -1;
@@ -1269,18 +1270,7 @@
nodeRecordMap.put(nodeId, currentNode);
}
- //DocumentIdentifier doc = null;
- //String doc = MetaCatUtil.getDocIdFromAccessionNumber(docid);
String doc = docid;
- /*try {
- doc = new DocumentIdentifier(docid);
- } catch (AccessionNumberException e){
- logMetacat.info("AccessionNumber Exception while "
- + " inserting path index "
- + "in DocumentImpl.buildIndex for "
- + "document " + docid, 10);
- logMetacat.info(e.getMessage(), 10);
- }*/
// Opening separate db connection for deleting and writing
// XML Index -- be sure that it is all in one db transaction
@@ -1296,30 +1286,15 @@
// Delete the previous index entries for this document
deleteNodeIndex(dbConn);
- // Delete all the entries in xml_queryresult
- PreparedStatement pstmt = dbConn.prepareStatement(
- "DELETE FROM xml_queryresult WHERE docid = ?");
- pstmt.setString(1, doc);
- pstmt.execute();
- pstmt.close();
- dbConn.increaseUsageCount(1);
-
- // Delete all the entries in xml_queryresult
- pstmt = dbConn.prepareStatement(
- "DELETE FROM xml_path_index WHERE docid = ?");
- pstmt.setString(1, doc);
- pstmt.execute();
- pstmt.close();
- dbConn.increaseUsageCount(1);
- //System.out.println("after dleteing !!!!!!!!!!!1");
// Step through all of the node records we were given
// and build the new index and update the database
it = nodeRecordLists.iterator();
+ Vector pathsFound = new Vector();;
while (it.hasNext()) {
NodeRecord currentNode = (NodeRecord) it.next();
HashMap pathList = new HashMap();
- if (currentNode.nodetype.equals("ELEMENT") ||
- currentNode.nodetype.equals("ATTRIBUTE") ) {
+ if (currentNode.getNodeType().equals("ELEMENT") ||
+ currentNode.getNodeType().equals("ATTRIBUTE")){
if (atRootElement) {
rootNodeId = currentNode.getNodeId();
@@ -1327,14 +1302,21 @@
}
traverseParents(nodeRecordMap, rootNodeId,
currentNode.getNodeId(),
- currentNode.getNodeId(), "", pathList);
+ currentNode.getNodeId(),
+ "", pathList, pathsFound);
updateNodeIndex(dbConn, pathList);
+ } else if (currentNode.getNodeType().equals("TEXT")){
+ if(!currentNode.getNodeData().trim().equals("")
+ && !pathsFound.isEmpty()){
+ updatePathIndex(dbConn, currentNode, pathsFound);
+ pathsFound.removeAllElements();
+ }
}
}
-
+
dbConn.commit();
} catch (SQLException e) {
- logMetacat.error("SQL Exception while inserting path index "
+ logMetacat.error("SQL Exception while indexing "
+ "in DocumentImpl.buildIndex for "
+ "document " + docid);
logMetacat.error(e.getMessage());
@@ -1349,92 +1331,8 @@
} finally {
DBConnectionPool.returnDBConnection(dbConn, serialNumber);
}
-
- try {
- dbConn = DBConnectionPool.getDBConnection("DocumentImpl.buildIndex-pathIndex");
- serialNumber = dbConn.getCheckOutSerialNumber();
- dbConn.setAutoCommit(false);
-
- ResultSet rs = null;
- PreparedStatement pstmt1 = null;
-
- String pathList = "";
- for (int i = 0; i < MetaCatUtil.pathsForIndexing.size(); i++) {
- pathList = pathList + "i.path = '" +
- (String)MetaCatUtil.pathsForIndexing.get(i) + "' OR ";
- }
-
- pathList = pathList.substring(0,pathList.length()-4);
-
- PreparedStatement pstmt = dbConn.prepareStatement("SELECT DISTINCT n.docid, "
- + "n.nodedata, n.nodedatanumerical,"
- + " n.parentnodeid, i.path FROM"
- + " xml_nodes n, xml_index i WHERE"
- + " (" + pathList
- + " ) AND n.parentnodeid=i.nodeid AND"
- + " n.nodetype LIKE 'TEXT' and n.docid = ?"
- + " order by n.parentnodeid");
-
- pstmt.setString(1, doc);
-
- pstmt.execute();
- rs = pstmt.getResultSet();
- dbConn.increaseUsageCount(1);
-
- int count = 0;
-
- while (rs.next()) {
- //System.out.println("update xml_path_index!!!!!!!1");
- String docid = rs.getString(1);
- String nodedata = rs.getString(2);
- float nodedatanumerical = rs.getFloat(3);
- int parentnodeid = rs.getInt(4);
- String path = rs.getString(5);
-
- if (!nodedata.trim().equals("")) {
- pstmt1 = dbConn.prepareStatement("INSERT INTO "
- + "xml_path_index (docid, path, nodedata, "
- + "nodedatanumerical, parentnodeid)"
- + " VALUES (?, ?, ?, ?, ?)");
-
- pstmt1.setString(1, docid);
- pstmt1.setString(2, path);
- pstmt1.setString(3, nodedata);
- pstmt1.setFloat(4, nodedatanumerical);
- pstmt1.setInt(5, parentnodeid);
-
- pstmt1.execute();
- pstmt1.close();
- dbConn.increaseUsageCount(1);
-
- count++;
- }
- }
-
- rs.close();
- pstmt.close();
- dbConn.increaseUsageCount(1);
-
- logMetacat.warn("Indexed " + count
- + " records for docid: " + docid);
-
- dbConn.commit();
- } catch (SQLException e) {
- logMetacat.error("SQL Exception while inserting path index "
- + "in DocumentImpl.buildIndex for "
- + "document " + docid);
- logMetacat.error(e.getMessage());
- try {
- dbConn.rollback();
- } catch (SQLException sqle) {
- logMetacat.error("Error while rolling back "
- + "commit in DocumentImpl"
- + ".buildIndex" + "\n"
- + sqle.getMessage());
- }
- } finally {
- DBConnectionPool.returnDBConnection(dbConn, serialNumber);
- }
+
+ logMetacat.warn("BuildIndex complete for docid " + docid);
}
/**
@@ -1451,41 +1349,54 @@
*/
private void traverseParents(HashMap records, long rootNodeId,
long leafNodeId, long id,
- String children, HashMap pathList) {
- Long nodeId = new Long(id);
+ String children, HashMap pathList, Vector pathsFoundForIndexing) {
+ Long nodeId = new Long(id);
NodeRecord current = (NodeRecord)records.get(nodeId);
long parentId = current.getParentNodeId();
String currentName = current.getNodeName();
- if (current.nodetype.equals("ELEMENT") ||
- current.nodetype.equals("ATTRIBUTE") ) {
-
+ if (current.getNodeType().equals("ELEMENT") ||
+ current.getNodeType().equals("ATTRIBUTE") ) {
+
if (children.equals("")) {
- if (current.nodetype.equals("ATTRIBUTE")) {
+ if (current.getNodeType().equals("ATTRIBUTE")) {
currentName = "@" + currentName;
}
logMetacat.debug("A: " + currentName +"\n");
pathList.put(currentName, new PathIndexEntry(leafNodeId,
currentName, docid, doctype, parentId));
+ if(MetaCatUtil.pathsForIndexing.contains(currentName)){
+ logMetacat.debug("paths found " + currentName);
+ pathsFoundForIndexing.add(currentName);
+ }
}
currentName = "/" + currentName;
currentName = currentName + children;
if (parentId != 0) {
traverseParents(records, rootNodeId, leafNodeId,
- parentId, currentName, pathList);
+ parentId, currentName, pathList, pathsFoundForIndexing);
}
String path = current.getNodeName() + children;
+
if (!children.equals("")) {
logMetacat.debug("B: " + path +"\n");
pathList.put(path, new PathIndexEntry(leafNodeId, path, docid,
doctype, parentId));
+ if(MetaCatUtil.pathsForIndexing.contains(path)){
+ logMetacat.debug("paths found " + path);
+ pathsFoundForIndexing.add(path);
+ }
}
if (id == rootNodeId) {
String fullPath = '/' + path;
logMetacat.debug("C: " + fullPath +"\n");
pathList.put(fullPath, new PathIndexEntry(leafNodeId, fullPath,
docid, doctype, parentId));
+ if(MetaCatUtil.pathsForIndexing.contains(fullPath)){
+ logMetacat.debug("paths found " + fullPath);
+ pathsFoundForIndexing.add(fullPath);
+ }
}
- }
+ }
}
/**
@@ -1514,6 +1425,27 @@
pstmt.close();
logMetacat.info("Deleted " + rows + " rows from xml_index " +
"for document " + docid);
+
+ // Delete all the entries in xml_queryresult
+ pstmt = conn.prepareStatement(
+ "DELETE FROM xml_queryresult WHERE docid = ?");
+ pstmt.setString(1, docid);
+ rows = pstmt.executeUpdate();
+ conn.increaseUsageCount(1);
+ pstmt.close();
+ logMetacat.info("Deleted " + rows + " rows from xml_queryresult " +
+ "for document " + docid);
+
+ // Delete all the entries in xml_path_index
+ pstmt = conn.prepareStatement(
+ "DELETE FROM xml_path_index WHERE docid = ?");
+ pstmt.setString(1, docid);
+ rows = pstmt.executeUpdate();
+ conn.increaseUsageCount(1);
+ pstmt.close();
+ logMetacat.info("Deleted " + rows + " rows from xml_path_index " +
+ "for document " + docid);
+
}
/**
@@ -1541,13 +1473,45 @@
// Step through the hashtable and insert each of the path values
Iterator it = pathList.values().iterator();
while (it.hasNext()) {
- PathIndexEntry entry = (PathIndexEntry)it.next();
- logMetacat.debug("Inserting: " + entry.nodeId +
- " (" + entry.parentId + "): " + entry.path);
- pstmt.setLong(1, entry.nodeId);
- pstmt.setString(2, entry.path);
- pstmt.setLong(5, entry.parentId);
- pstmt.executeUpdate();
+ PathIndexEntry entry = (PathIndexEntry)it.next();
+ logMetacat.debug("Inserting: " + entry.nodeId +
+ " (" + entry.parentId + "): " + entry.path);
+ pstmt.setLong(1, entry.nodeId);
+ pstmt.setString(2, entry.path);
+ pstmt.setLong(5, entry.parentId);
+ pstmt.executeUpdate();
+ }
+ // Close the database statement
+ pstmt.close();
+ }
+
+ /**
+ * Insert the paths from the pathList into the xml_path_index table on the
+ * database.
+ *
+ * @param conn the database connection to use, keeping a single transaction
+ * @param pathList the hash of paths to insert
+ * @throws SQLException if there is an error inserting into the db
+ */
+ private void updatePathIndex(DBConnection conn, NodeRecord currentNode, Vector pathList)
+ throws SQLException {
+ // Create an insert statement to reuse for all of the path
+ // insertions
+ PreparedStatement pstmt = conn.prepareStatement("INSERT INTO "
+ + "xml_path_index (docid, path, nodedata, "
+ + "nodedatanumerical, parentnodeid)"
+ + " VALUES (?, ?, ?, ?, ?)");
+ pstmt.setString(1, docid);
+ // Step through the hashtable and insert each of the path values
+ Iterator it = pathList.iterator();
+ while (it.hasNext()) {
+ String path = (String)it.next();
+ logMetacat.debug("Inserting: "+ path);
+ pstmt.setString(2, path);
+ pstmt.setString(3, currentNode.getNodeData());
+ pstmt.setFloat(4, currentNode.getNodeDataNumerical());
+ pstmt.setLong(5, currentNode.getParentNodeId());
+ pstmt.execute();
}
// Close the database statement
pstmt.close();
@@ -1777,9 +1741,9 @@
String nodename = null;
String nodeprefix = null;
String nodedata = null;
- String quotechar = dbAdapter.getStringDelimiter();
+ float nodedatanumerical = -1;
String sql = "SELECT nodeid,parentnodeid,nodeindex, "
- + "nodetype,nodename,nodeprefix,nodedata "
+ + "nodetype,nodename,nodeprefix,nodedata,nodedatanumerical "
+ "FROM xml_nodes WHERE rootnodeid = ?";
// go through the access control for some nodes
@@ -1815,9 +1779,11 @@
nodeprefix = rs.getString(6);
nodedata = rs.getString(7);
nodedata = MetaCatUtil.normalize(nodedata);
+ nodedatanumerical = rs.getFloat(8);
+
// add the data to the node record list hashtable
NodeRecord currentRecord = new NodeRecord(nodeid, parentnodeid,
- nodeindex, nodetype, nodename, nodeprefix, nodedata);
+ nodeindex, nodetype, nodename, nodeprefix, nodedata, nodedatanumerical);
nodeRecordList.add(currentRecord);
// Advance to the next node
@@ -1869,6 +1835,7 @@
String nodename = null;
String nodeprefix = null;
String nodedata = null;
+ float nodedatanumerical = -1;
String quotechar = dbAdapter.getStringDelimiter();
String table = "xml_nodes";
//System.out.println("in getNodeREcorelist !!!!!!!!!!!for root node id "+rootnodeid);
@@ -1892,7 +1859,7 @@
serialNumber = dbconn.getCheckOutSerialNumber();
pstmt = dbconn
.prepareStatement("SELECT nodeid,parentnodeid,nodeindex, "
- + "nodetype,nodename,nodeprefix,nodedata "
+ + "nodetype,nodename,nodeprefix,nodedata, nodedatanumerical "
+ "FROM " + table + " WHERE rootnodeid = ?");
// Bind the values to the query
@@ -1911,10 +1878,12 @@
nodename = rs.getString(5);
nodeprefix = rs.getString(6);
nodedata = rs.getString(7);
- nodedata = MetaCatUtil.normalize(nodedata);
+ nodedata = MetaCatUtil.normalize(nodedata);
+ nodedatanumerical = rs.getFloat(8);
+
// add the data to the node record list hashtable
NodeRecord currentRecord = new NodeRecord(nodeid, parentnodeid,
- nodeindex, nodetype, nodename, nodeprefix, nodedata);
+ nodeindex, nodetype, nodename, nodeprefix, nodedata, nodedatanumerical);
nodeRecordList.add(currentRecord);
// Advance to the next node
More information about the Metacat-cvs
mailing list