[freenet-dev] [freenet-cvs] r19675 - in trunk/freenet/src/freenet: crypt node store support
Matthew Toseland
toad at amphibian.dyndns.org
Sat May 3 19:04:05 UTC 2008
On Friday 02 May 2008 11:10, j16sdiz at freenetproject.org wrote:
> Author: j16sdiz
> Date: 2008-05-02 10:10:12 +0000 (Fri, 02 May 2008)
> New Revision: 19675
>
> Modified:
> trunk/freenet/src/freenet/crypt/SHA256.java
> trunk/freenet/src/freenet/node/MemoryChecker.java
> trunk/freenet/src/freenet/node/Node.java
> trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> trunk/freenet/src/freenet/support/OOMHandler.java
> trunk/freenet/src/freenet/support/OOMHook.java
> Log:
> alternative low memory / out of memory handling
>
> Modified: trunk/freenet/src/freenet/crypt/SHA256.java
> ===================================================================
> --- trunk/freenet/src/freenet/crypt/SHA256.java 2008-05-02 10:09:13 UTC (rev
19674)
> +++ trunk/freenet/src/freenet/crypt/SHA256.java 2008-05-02 10:10:12 UTC (rev
19675)
> @@ -44,6 +44,8 @@
> import freenet.node.Node;
> import freenet.node.NodeInitException;
> import freenet.support.Logger;
> +import freenet.support.OOMHandler;
> +import freenet.support.OOMHook;
> import freenet.support.io.Closer;
>
> /**
> @@ -105,7 +107,7 @@
> String algo = md256.getAlgorithm();
> if(!(algo.equals("SHA-256") || algo.equals("SHA256")))
> throw new IllegalArgumentException("Should be SHA-256 but is " + algo);
> - if (digests.size() > 16) // don't cache too many of them
> + if (digests.size() > 16 || noCache) // don't cache too many of them
> return;
> md256.reset();
> digests.add(md256);
> @@ -121,4 +123,20 @@
> public static int getDigestLength() {
> return HASH_SIZE;
> }
> +
> + private static boolean noCache = false;
> +
> + static {
> + OOMHandler.addOOMHook(new OOMHook() {
> + public void handleLowMemory() throws Exception {
> + digests.clear();
> + noCache = true;
> + }
> +
> + public void handleOutOfMemory() throws Exception {
> + digests.clear();
> + noCache = true;
> + }
> + });
> + }
> }
So when we reach the memory limit, we turn off caching and get more churn and
therefore hit it more quickly ... well I suppose it's a tradeoff.
>
> Modified: trunk/freenet/src/freenet/node/MemoryChecker.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/MemoryChecker.java 2008-05-02 10:09:13
UTC (rev 19674)
> +++ trunk/freenet/src/freenet/node/MemoryChecker.java 2008-05-02 10:10:12
UTC (rev 19675)
> @@ -4,6 +4,7 @@
> package freenet.node;
>
> import freenet.support.Logger;
> +import freenet.support.OOMHandler;
> import freenet.support.SizeUtil;
>
> public class MemoryChecker implements Runnable {
> @@ -42,6 +43,11 @@
>
> Logger.normal(this, "Memory in
use: "+SizeUtil.formatSize((r.totalMemory()-r.freeMemory())));
>
> + if (r.freeMemory() < 4096 * 1024 * 1024) { // free memory < 8 MB
> + Logger.error(this, "memory too low, trying to free some");
> + OOMHandler.lowMemory();
> + }
> +
I've changed this to do a GC and then re-check. If we are still at the limit
for total memory and have less than 8MB free, *then* we hit the low memory
handler.
I'm not entirely convinced though. We will end up doing a lot more full GCs
this way ... often on systems that don't have a memory problem.
> int sleeptime = aggressiveGCModificator;
> if(sleeptime <= 0) { // We are done
> ps.queueTimedJob(this, 120 * 250); // 30 sec
>
> Modified: trunk/freenet/src/freenet/node/Node.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/Node.java 2008-05-02 10:09:13 UTC (rev
19674)
> +++ trunk/freenet/src/freenet/node/Node.java 2008-05-02 10:10:12 UTC (rev
19675)
> @@ -124,7 +124,7 @@
> /**
> * @author amphibian
> */
> -public class Node implements TimeSkewDetectorCallback, GetPubkey, OOMHook {
> +public class Node implements TimeSkewDetectorCallback, GetPubkey {
>
> private static boolean logMINOR;
>
> @@ -1604,8 +1604,6 @@
> e.printStackTrace();
> throw new
NodeInitException(NodeInitException.EXIT_COULD_NOT_START_UPDATER, "Could not
create Updater: "+e);
> }
> -
> - OOMHandler.addOOMHook(this);
>
> Logger.normal(this, "Node constructor completed");
> System.out.println("Node constructor completed");
> @@ -3306,23 +3304,4 @@
> public void setDispatcherHook(NodeDispatcherCallback cb) {
> this.dispatcher.setHook(cb);
> }
> -
> - /**
> - * Free some memory
> - */
> - public void handleOOM() throws Exception {
> - if (cachedPubKeys != null) {
> - Object value;
> - do {
> - value = cachedPubKeys.popKey();
> - } while (value != null);
> - }
> - if (recentlyCompletedIDs != null) {
> - synchronized (recentlyCompletedIDs) {
> - // half it size
> - while (recentlyCompletedIDs.size() > MAX_RECENTLY_COMPLETED_IDS / 2)
> - recentlyCompletedIDs.pop();
> - }
> - }
> - }
> }
Why did you delete the pubkey cache clearing OOM hook?
>
> Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
> ===================================================================
> --- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2008-05-02
10:09:13 UTC (rev 19674)
> +++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2008-05-02
10:10:12 UTC (rev 19675)
> @@ -2239,15 +2239,30 @@
> bf.get(data);
> }
>
> - public void handleOOM() throws Exception {
> - if (storeRAF != null)
> - storeRAF.getFD().sync();
> - if (keysRAF != null)
> - keysRAF.getFD().sync();
> - if (lruRAF != null)
> - lruRAF.getFD().sync();
> + public void handleLowMemory() throws Exception {
> + // Flush all
> + if (storeFC != null)
> + storeFC.force(true);
> + if (keysFC != null)
> + keysFC.force(true);
> + if (lruFC != null)
> + lruFC.force(true);
> }
>
> + public void handleOutOfMemory() throws Exception {
> + // database likely to be corrupted,
> + // reconstruct it just in case
> + reconstructFile.createNewFile();
> +
> + // Flush all
> + if (storeFC != null)
> + storeFC.force(true);
> + if (keysFC != null)
> + keysFC.force(true);
> + if (lruFC != null)
> + lruFC.force(true);
> + }
> +
> /**
> * @return
> */
>
> Modified: trunk/freenet/src/freenet/support/OOMHandler.java
> ===================================================================
> --- trunk/freenet/src/freenet/support/OOMHandler.java 2008-05-02 10:09:13
UTC (rev 19674)
> +++ trunk/freenet/src/freenet/support/OOMHandler.java 2008-05-02 10:10:12
UTC (rev 19675)
> @@ -33,6 +33,30 @@
> }
> }
>
> + /**
> + * Call this when running low of memory
> + */
> + public static void lowMemory() {
> + System.gc();
> + System.runFinalization();
> +
> + // iterate all oom hooks
> + Iterator it = oomHooks.iterator();
> + while (it.hasNext()) {
> + OOMHook hook = ((OOMHook) it.next());
> + if (hook != null) {
> + try {
> + hook.handleLowMemory();
> + } catch (Throwable t) {
> + //ignore
> + }
> + }
> + }
> +
> + System.gc();
> + System.runFinalization();
> + }
> +
> public static void handleOOM(OutOfMemoryError e) {
> if (isOOM) {
> Logger.error(null, "Double OOM", e);
> @@ -51,20 +75,18 @@
>
> System.gc();
> System.runFinalization();
> -
> +
> // iterate all oom hooks
> Iterator it = oomHooks.iterator();
> while (it.hasNext()) {
> OOMHook hook = ((OOMHook) it.next());
> if (hook != null) {
> try {
> - hook.handleOOM();
> + hook.handleOutOfMemory();
> } catch (Throwable t) {
> //ignore
> }
> }
> -
> - System.gc();
> }
>
> System.gc();
>
> Modified: trunk/freenet/src/freenet/support/OOMHook.java
> ===================================================================
> --- trunk/freenet/src/freenet/support/OOMHook.java 2008-05-02 10:09:13 UTC
(rev 19674)
> +++ trunk/freenet/src/freenet/support/OOMHook.java 2008-05-02 10:10:12 UTC
(rev 19675)
> @@ -5,9 +5,14 @@
> */
> public interface OOMHook {
> /**
> - * Handle OutOfMemoryError
> + * Handle running low of memory
> *
> * (try to free some cache, save the files, etc).
> */
> - void handleOOM() throws Exception;
> + void handleLowMemory() throws Exception;
> +
> + /**
> + * Handle running out of memory
> + */
> + void handleOutOfMemory() throws Exception;
> }
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080503/bd149707/attachment.pgp
More information about the Devl
mailing list