svn commit: r437511 - in /jakarta/jcs/trunk/src: java/org/apache/jcs/auxiliary/disk/block/ test-conf/ test/org/apache/jcs/auxiliary/disk/block/

Author: asmuts
Date: Sun Aug 27 19:29:54 2006
New Revision: 437511

URL: http://svn.apache.org/viewvc?rev=437511&view=rev
Log:
Added configurable, periodic key persistence to the block disk cache.

Added rough disk file verification to block disk cache.


Modified:
    
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
    
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
    
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
    
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
    jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf
    
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java

Modified: 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java 
(original)
+++ 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java 
Sun Aug 27 19:29:54 2006
@@ -64,8 +64,13 @@
      */
     public BlockDisk( File file )
         throws FileNotFoundException
-    {
+    {        
         this( file, DEFAULT_BLOCK_SIZE_BYTES );
+        if ( log.isInfoEnabled() )
+        {
+            log.info( "Used default block size [" + DEFAULT_BLOCK_SIZE_BYTES + 
"]" );
+        }
+        
     }
 
     /**

Modified: 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
 (original)
+++ 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
 Sun Aug 27 19:29:54 2006
@@ -92,10 +92,9 @@
 
         try
         {
-            if ( cacheAttributes.getBlockSizeBytes() > 0 )
+            if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 )
             {
-                this.dataFile = new BlockDisk( new File( rootDirectory, 
fileName + ".data" ), cacheAttributes
-                    .getBlockSizeBytes() );
+                this.dataFile = new BlockDisk( new File( rootDirectory, 
fileName + ".data" ), this.blockDiskCacheAttributes.getBlockSizeBytes() );
             }
             else
             {
@@ -104,7 +103,9 @@
 
             keyStore = new BlockDiskKeyStore( this.blockDiskCacheAttributes, 
this );
 
-            if ( keyStore.size() == 0 )
+            boolean alright = verifyDisk();
+
+            if ( keyStore.size() == 0 || !alright )
             {
                 this.reset();
             }
@@ -113,7 +114,7 @@
             alive = true;
             if ( log.isInfoEnabled() )
             {
-                log.info( logCacheName + "Indexed Disk Cache is alive." );
+                log.info( logCacheName + "Block Disk Cache is alive." );
             }
         }
         catch ( Exception e )
@@ -126,6 +127,43 @@
     }
 
     /**
+     * We need to verify that the file on disk uses the same block size and 
that the file is the
+     * proper size.
+     * <p>
+     * @return true if it looks ok
+     */
+    protected boolean verifyDisk()
+    {
+        boolean alright = false;
+        // simply try to read a few. If it works, then the file is probably ok.
+        // TODO add more.
+        try
+        {
+            int maxToTest = 100;
+            int count = 0;
+            Set keySet = this.keyStore.entrySet();
+            Iterator it = keySet.iterator();
+            while ( it.hasNext() && count < maxToTest )
+            {
+                count++;
+                Map.Entry entry = (Map.Entry) it.next();
+                Object data = this.dataFile.read( (int[]) entry.getValue() );
+                if ( data == null )
+                {
+                    throw new Exception( "Couldn't find data for key [" + 
entry.getKey() + "]" );
+                }
+            }
+            alright = true;
+        }
+        catch ( Exception e )
+        {
+            log.warn( "Problem verifying disk.  Message [" +  e.getMessage() + 
"]" );
+            alright = false;
+        }
+        return alright;
+    }
+
+    /**
      * This requires a full iteration through the keys.
      * <p>
      * (non-Javadoc)
@@ -459,7 +497,6 @@
         storageLock.writeLock().acquire();
         try
         {
-
             // Prevents any interaction with the cache while we're shutting 
down.
             alive = false;
 
@@ -472,7 +509,7 @@
                     log.debug( logCacheName + "Closing files, base filename: " 
+ fileName );
                 }
                 dataFile.close();
-                dataFile = null;
+                // dataFile = null;
 
                 // TOD make a close
                 // keyFile.close();
@@ -527,8 +564,15 @@
             File dataFileTemp = new File( this.rootDirectory, fileName + 
".data" );
             dataFileTemp.delete();
 
-            dataFile = new BlockDisk( new File( this.rootDirectory, fileName + 
".data" ) );
-
+            if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 )
+            {
+                this.dataFile = new BlockDisk( new File( rootDirectory, 
fileName + ".data" ), this.blockDiskCacheAttributes.getBlockSizeBytes() );
+            }
+            else
+            {
+                this.dataFile = new BlockDisk( new File( rootDirectory, 
fileName + ".data" ) );
+            }
+            
             this.keyStore.reset();
         }
         catch ( Exception e )

Modified: 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
 (original)
+++ 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
 Sun Aug 27 19:29:54 2006
@@ -19,6 +19,12 @@
     /** -1 means no limit. */
     private int maxKeySize = DEFAULT_MAX_KEY_SIZE;
     
+    private static final long DEFAULT_KEY_PERSISTENCE_INTERVAL_SECONDS = 5 * 
60;
+    
+    /** The keys will be persisted at this interval.  -1 mean never. */
+    private long keyPersistenceIntervalSeconds = 
DEFAULT_KEY_PERSISTENCE_INTERVAL_SECONDS;
+    
+    
     /**
      * The size of the blocks. All blocks are the same size.
      * <p>
@@ -54,6 +60,22 @@
     }
 
     /**
+     * @param keyPersistenceIntervalSeconds The keyPersistenceIntervalSeconds 
to set.
+     */
+    public void setKeyPersistenceIntervalSeconds( long 
keyPersistenceIntervalSeconds )
+    {
+        this.keyPersistenceIntervalSeconds = keyPersistenceIntervalSeconds;
+    }
+
+    /**
+     * @return Returns the keyPersistenceIntervalSeconds.
+     */
+    public long getKeyPersistenceIntervalSeconds()
+    {
+        return keyPersistenceIntervalSeconds;
+    }
+
+    /**
      * Write out the values for debugging purposes.
      * <p>
      * @return String
@@ -66,6 +88,7 @@
         str.append( "\n MaxKeySize [" + this.getMaxKeySize() + "]" );
         str.append( "\n MaxPurgatorySize [" + this.getMaxPurgatorySize() + "]" 
);
         str.append( "\n BlockSizeBytes [" + this.getBlockSizeBytes() + "]" );
+        str.append( "\n KeyPersistenceIntervalSeconds [" + 
this.getKeyPersistenceIntervalSeconds() + "]" );
         return str.toString();
     }
 }

Modified: 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
 (original)
+++ 
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
 Sun Aug 27 19:29:54 2006
@@ -28,6 +28,9 @@
 import org.apache.jcs.auxiliary.disk.LRUMapJCS;
 import org.apache.jcs.utils.timing.ElapsedTimer;
 
+import EDU.oswego.cs.dl.util.concurrent.ClockDaemon;
+import EDU.oswego.cs.dl.util.concurrent.ThreadFactory;
+
 /**
  * This is responsible for storing the keys.
  * <p>
@@ -35,7 +38,7 @@
  */
 public class BlockDiskKeyStore
 {
-    private static final Log log = LogFactory.getLog( BlockDiskCache.class );
+    private static final Log log = LogFactory.getLog( BlockDiskKeyStore.class 
);
 
     private BlockDiskCacheAttributes blockDiskCacheAttributes;
 
@@ -55,6 +58,11 @@
     private File rootDirectory;
 
     /**
+     * The background key persister, one for all regions.
+     */
+    private static ClockDaemon persistenceDaemon;
+
+    /**
      * Set the configuration options.
      * <p>
      * @param cacheAttributes
@@ -95,6 +103,26 @@
         {
             initKeyMap();
         }
+
+        // add this region to the persistence thread.
+        // TODO we might need to stagger this a bit.
+        if ( this.blockDiskCacheAttributes.getKeyPersistenceIntervalSeconds() 
> 0 )
+        {
+            if ( persistenceDaemon == null )
+            {
+                persistenceDaemon = new ClockDaemon();
+                persistenceDaemon.setThreadFactory( new MyThreadFactory() );
+            }
+            persistenceDaemon
+                .executePeriodically( 
this.blockDiskCacheAttributes.getKeyPersistenceIntervalSeconds() * 1000,
+                                      new Runnable()
+                                      {
+                                          public void run()
+                                          {
+                                              saveKeys();
+                                          }
+                                      }, false );
+        }
     }
 
     /**
@@ -369,6 +397,27 @@
                 log.debug( logCacheName + "Removing key: [" + key + "] from 
key store." );
                 log.debug( logCacheName + "Key store size: [" + this.size() + 
"]." );
             }
+        }
+    }
+
+    /**
+     * Allows us to set the daemon status on the clockdaemon
+     * @author aaronsm
+     */
+    class MyThreadFactory
+        implements ThreadFactory
+    {
+
+        /*
+         * (non-Javadoc)
+         * @see 
EDU.oswego.cs.dl.util.concurrent.ThreadFactory#newThread(java.lang.Runnable)
+         */
+        public Thread newThread( Runnable runner )
+        {
+            Thread t = new Thread( runner );
+            t.setDaemon( true );
+            t.setPriority( Thread.MIN_PRIORITY );
+            return t;
         }
     }
 }

Modified: jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf (original)
+++ jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf Sun Aug 27 
19:29:54 2006
@@ -20,9 +20,9 @@
 # Block Disk Cache
 
jcs.auxiliary.blockDiskCache=org.apache.jcs.auxiliary.disk.block.BlockDiskCacheFactory
 
jcs.auxiliary.blockDiskCache.attributes=org.apache.jcs.auxiliary.disk.block.BlockDiskCacheAttributes
-jcs.auxiliary.blockDiskCache.attributes.DiskPath=target/test-sandbox/block-disk-cache-conc
+jcs.auxiliary.blockDiskCache.attributes.DiskPath=target/test-sandbox/block-disk-cache-huge
 jcs.auxiliary.blockDiskCache.attributes.MaxPurgatorySize=300000
-jcs.auxiliary.blockDiskCache.attributes.MaxKeySize=500000
+jcs.auxiliary.blockDiskCache.attributes.MaxKeySize=1000000
 jcs.auxiliary.blockDiskCache.attributes.blockSizeBytes=500
 jcs.auxiliary.blockDiskCache.attributes.EventQueueType=SINGLE
 
#jcs.auxiliary.blockDiskCache.attributes.EventQueuePoolName=disk_cache_event_queue

Modified: 
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
URL: 
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- 
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
 (original)
+++ 
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
 Sun Aug 27 19:29:54 2006
@@ -3,6 +3,8 @@
 import junit.framework.TestCase;
 
 import org.apache.jcs.JCS;
+import org.apache.jcs.utils.timing.ElapsedTimer;
+import org.apache.jcs.utils.timing.SleepUtil;
 
 /**
  * Put a few hundred thousand entries in the block disk cache.
@@ -38,15 +40,19 @@
         int items = 300000;
         String region = "testCache1";
 
-        JCS jcs = JCS.getInstance( region );
+        System.out.println( "--------------------------" );
+        long initialMemory = measureMemoryUse();
+        System.out.println( "Before getting JCS: " + initialMemory );
 
+        JCS jcs = JCS.getInstance( region );
+        jcs.clear();
+        
         try
         {
-
+            ElapsedTimer timer = new ElapsedTimer();            
             System.out.println( "Start: " + measureMemoryUse() );
 
             // Add items to cache
-
             for ( int i = 0; i <= items; i++ )
             {
                 jcs.put( i + ":key", region + " data " + i );
@@ -61,24 +67,40 @@
             System.out.println( jcs.getStats() );
             System.out.println( "--------------------------" );
             System.out.println( "After wait: " + measureMemoryUse() );
+            
+            for ( int i = 0; i < 10; i++ )
+            {
+                SleepUtil.sleepAtLeast( 3000 );
+                System.out.println( "--------------------------" );
+                System.out.println( "After sleep. " + 
timer.getElapsedTimeString() + " memory used = " + measureMemoryUse() );
+                System.out.println( jcs.getStats() );                
+            }
 
             // Test that all items are in cache
-
+            System.out.println( "--------------------------" );
+            System.out.println( "Retrieving all." );
             for ( int i = 0; i <= items; i++ )
             {
+                //System.out.print(  "\033[s" );
                 String value = (String) jcs.get( i + ":key" );
-
-                assertEquals( region + " data " + i, value );
+                if( i % 1000 == 0 )
+                {
+                    //System.out.print(  "\033[r" );
+                    System.out.println(  i + " ");
+                }
+                assertEquals( "Wrong value returned.", region + " data " + i, 
value );
             }
-
-            System.out.println( "After get: " + measureMemoryUse() );
+            long aftetGet = measureMemoryUse();
+            System.out.println( "After get: " + aftetGet + " diff = " + 
(aftetGet - initialMemory));
+        
         }
         finally
         {
             // dump the stats to the report
             System.out.println( jcs.getStats() );
             System.out.println( "--------------------------" );
-            System.out.println( "End: " + measureMemoryUse() );
+            long endMemory = measureMemoryUse();
+            System.out.println( "End: " + endMemory + " diff = " + (endMemory 
- initialMemory) );
         }
     }