[freenet-cvs] r13635 - trunk/freenet/src/freenet/clients/http

[swatig0 at freenetproject.org]

Author: swatig0
Date: 2007-06-17 11:42:50 +0000 (Sun, 17 Jun 2007)
New Revision: 13635

Modified:
   trunk/freenet/src/freenet/clients/http/XMLSpider.java
Log:
The new XML spider puts file ids in the subindex

Modified: trunk/freenet/src/freenet/clients/http/XMLSpider.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/XMLSpider.java	2007-06-17 11:42:03 UTC (rev 13634)
+++ trunk/freenet/src/freenet/clients/http/XMLSpider.java	2007-06-17 11:42:50 UTC (rev 13635)
@@ -83,7 +83,7 @@
 	private static final int minTimeBetweenEachIndexRewriting = 1;
 	//private static final String indexFilename = "index.xml";
 	private static final String DEFAULT_INDEX_DIR = "/home/swati/myindex/";
-	private static final int MAX_ENTRIES = 50;
+	private static final int MAX_ENTRIES = 5;
 	private static final String pluginName = "XML spider";
 	
 	private static final String indexTitle= "This is an index";
@@ -112,9 +112,9 @@
 	}
 
 	private void startSomeRequests() {
-		try{
-			Thread.sleep(30 * 1000); // Let the node start up
-		} catch (InterruptedException e){}
+//		try{
+//			Thread.sleep(30 * 1000); // Let the node start up
+//		} catch (InterruptedException e){}
 		
 		FreenetURI[] initialURIs = core.bookmarkManager.getBookmarkURIs();
 		for (int i = 0; i < initialURIs.length; i++)
@@ -420,31 +420,31 @@
 		urisToNumbers = new HashMap();
 		Element prefixElement = xmlDoc.createElement("prefix");
 		prefixElement.setAttribute("value", prefix+"");
-		Element filesElement = xmlDoc.createElement("files"); /* filesElement != fileElement */
+	//	Element filesElement = xmlDoc.createElement("files"); /* filesElement != fileElement */
 
 		for (int i = 0; i < uris.length; i++) {
 			urisToNumbers.put(uris[i], new Integer(i));
 			
-			Element fileElement = xmlDoc.createElement("file");
-
-			fileElement.setAttribute("id", Integer.toString(i));
-			fileElement.setAttribute("key", uris[i].toString());
-			
-			Long size = (Long)sizeOfURIs.get(uris[i].toString());
-
-			if(size == null) {
-				Logger.error(this, "Spider: size is missing");
-			} else {
-				fileElement.setAttribute("size", size.toString());
-			}
-			fileElement.setAttribute("mime", ((String)mimeOfURIs.get(uris[i].toString())));
-
-			Element titleElement = xmlDoc.createElement("option");
-			titleElement.setAttribute("name", "title");
-			titleElement.setAttribute("value", (String)titlesOfURIs.get(uris[i].toString()));
-
-			fileElement.appendChild(titleElement);
-			filesElement.appendChild(fileElement);
+//			Element fileElement = xmlDoc.createElement("file");
+//
+//			fileElement.setAttribute("id", Integer.toString(i));
+//			fileElement.setAttribute("key", uris[i].toString());
+//			
+//			Long size = (Long)sizeOfURIs.get(uris[i].toString());
+//
+//			if(size == null) {
+//				Logger.error(this, "Spider: size is missing");
+//			} else {
+//				fileElement.setAttribute("size", size.toString());
+//			}
+//			fileElement.setAttribute("mime", ((String)mimeOfURIs.get(uris[i].toString())));
+//
+//			Element titleElement = xmlDoc.createElement("option");
+//			titleElement.setAttribute("name", "title");
+//			titleElement.setAttribute("value", (String)titlesOfURIs.get(uris[i].toString()));
+//
+//			fileElement.appendChild(titleElement);
+//			filesElement.appendChild(fileElement);
 		}
 
 		
@@ -469,7 +469,7 @@
 		rootElement.appendChild(prefixElement);
 		rootElement.appendChild(headerElement);
 		
-		rootElement.appendChild(filesElement);
+		//rootElement.appendChild(filesElement);
 		rootElement.appendChild(keywordsElement);
 
 		/* Serialization */
@@ -512,10 +512,7 @@
 	private synchronized void generateIndex() throws Exception{
 		String[] words = (String[]) urisByWord.keySet().toArray(new String[urisByWord.size()]);
 		Arrays.sort(words);
-			 
-			
-				
-		for (int i = 0; i < 100; i++) {
+		for (int i = 0; i < words.length; i++) {
 		try{
 		
 		String prefix_match = getIndex(words[i]);
@@ -537,7 +534,7 @@
 			}
 			output2.close();
 	}
-		catch(Exception e2){ }
+		catch(Exception e2){Logger.error(this,"The Word could not be added"+ e2.toString(), e2); }
 		}	
 
 	
@@ -608,7 +605,8 @@
 			
 			Attr no_entries = (Attr) entry.getAttributes().getNamedItem("value");
 			
-			
+			Element filesElement = (Element) root.getElementsByTagName("files").item(0);
+			NodeList filesList = filesElement.getElementsByTagName("file");
 			if(Integer.parseInt(no_entries.getValue()) >= MAX_ENTRIES) return false;
 			else
 			{
@@ -632,8 +630,10 @@
 				}
 
 				Element uriElement = doc.createElement("file");
+				Element fileElement = doc.createElement("file");
 				uriElement.setAttribute("id", x.toString());
-//
+				fileElement.setAttribute("id", x.toString());
+				fileElement.setAttribute("key", uri.toString());
 //				/* Position by position */
 				HashMap positionsForGivenWord = (HashMap)positionsByWordByURI.get(uri.toString());
 				Integer[] positions = (Integer[])positionsForGivenWord.get(str);
@@ -648,8 +648,16 @@
 				}
 				
 				uriElement.appendChild(doc.createTextNode(positionList.toString()));
-
+				int l;
+			for(l = 0;l<filesList.getLength();l++)
+				{ Element file = (Element) filesList.item(l);
+				if(file.getAttribute("id").equals(x.toString()))
+				
+				break;
+				}
 				wordElement.appendChild(uriElement);
+				if(l>=filesList.getLength())
+				filesElement.appendChild(fileElement);
 			}
 			Element keywordsElement = (Element) root.getElementsByTagName("keywords").item(0);
 			keywordsElement.appendChild(wordElement);
@@ -686,7 +694,7 @@
 			return true;	
 		}
 		
-		catch(Exception e){}
+		catch(Exception e){Logger.error(this,"Word could not be added to the subindex"+ e.toString(), e);}
 		return false;
 	}
 	private void split(String prefix) throws Exception
@@ -928,6 +936,9 @@
 		
 		Thread starterThread = new Thread("Spider Plugin Starter") {
 			public void run() {
+				try{
+					Thread.sleep(30 * 1000); // Let the node start up
+				} catch (InterruptedException e){}
 				startSomeRequests();
 			}
 		};