Author: asmuts
Date: Sun Aug 27 11:20:07 2006
New Revision: 437431
URL: http://svn.apache.org/viewvc?rev=437431&view=rev
Log:
I made the block disk cache far more memory efficient.
It now only stores the block addresses in memory. The elememt descriptor is
used only for key persistence.
I made some initial optimizations on the serialization of the descriptor.
I added a unit test for the store.
Added:
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheKeyStoreUnitTest.java
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptor.java
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
jakarta/jcs/trunk/src/java/org/apache/jcs/utils/struct/LRUMap.java
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptorUnitTest.java
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskUnitTest.java
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
Sun Aug 27 11:20:07 2006
@@ -205,10 +205,10 @@
storageLock.readLock().acquire();
try
{
- BlockDiskElementDescriptor ded = this.keyStore.get( key );
+ int[] ded = this.keyStore.get( key );
if ( ded != null )
{
- object = (ICacheElement) this.dataFile.read(
ded.getBlocks() );
+ object = (ICacheElement) this.dataFile.read( ded );
}
}
finally
@@ -253,7 +253,7 @@
return;
}
- BlockDiskElementDescriptor old = null;
+ int[] old = null;
try
{
// make sure this only locks for one particular cache region
@@ -264,16 +264,12 @@
if ( old != null )
{
- this.dataFile.freeBlocks( old.getBlocks() );
+ this.dataFile.freeBlocks( old );
}
int[] blocks = this.dataFile.write( element );
- BlockDiskElementDescriptor newElement = new
BlockDiskElementDescriptor();
- newElement.setKey( element.getKey() );
- newElement.setBlocks( blocks );
-
- this.keyStore.put( element.getKey(), newElement );
+ this.keyStore.put( element.getKey(), blocks );
}
finally
{
@@ -333,8 +329,8 @@
if ( k instanceof String && k.toString().startsWith(
key.toString() ) )
{
- BlockDiskElementDescriptor ded = this.keyStore.get(
key );
- this.dataFile.freeBlocks( ded.getBlocks() );
+ int[] ded = this.keyStore.get( key );
+ this.dataFile.freeBlocks( ded );
iter.remove();
removed = true;
// TODO this needs to update the rmove count separately
@@ -352,8 +348,8 @@
if ( k instanceof GroupAttrName && ( (GroupAttrName) k
).groupId.equals( key ) )
{
- BlockDiskElementDescriptor ded = this.keyStore.get(
key );
- this.dataFile.freeBlocks( ded.getBlocks() );
+ int[] ded = this.keyStore.get( key );
+ this.dataFile.freeBlocks( ded );
iter.remove();
removed = true;
}
@@ -362,11 +358,11 @@
else
{
// remove single item.
- BlockDiskElementDescriptor ded = this.keyStore.remove( key );
+ int[] ded = this.keyStore.remove( key );
removed = ( ded != null );
if ( ded != null )
{
- this.dataFile.freeBlocks( ded.getBlocks() );
+ this.dataFile.freeBlocks( ded );
}
if ( log.isDebugEnabled() )
@@ -450,9 +446,10 @@
/**
* Internal method that handles the disposal.
- * @throws InterruptedException
+ * @throws InterruptedException
*/
- private void disposeInternal() throws InterruptedException
+ private void disposeInternal()
+ throws InterruptedException
{
if ( !alive )
{
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptor.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptor.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptor.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptor.java
Sun Aug 27 11:20:07 2006
@@ -1,27 +1,29 @@
package org.apache.jcs.auxiliary.disk.block;
/*
- * Copyright 2001-2004 The Apache Software Foundation. Licensed under the
Apache
- * License, Version 2.0 (the "License") you may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
- * or agreed to in writing, software distributed under the License is
- * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
+ * Copyright 2001-2004 The Apache Software Foundation. Licensed under the
Apache License, Version
+ * 2.0 (the "License") you may not use this file except in compliance with the
License. You may
+ * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by
+ * applicable law or agreed to in writing, software distributed under the
License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied. See
+ * the License for the specific language governing permissions and limitations
under the License.
*/
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import java.io.Serializable;
/**
- * This represents an element on disk. It must be kept small since these are
stored in memory. We
- * store the key and then offset in memory. That's all. We don't need the
length, since we don't
- * have a receyle bin.
+ * This represents an element on disk. This is used when we persist the keys.
We only store the
+ * block addresses in memory. We don't need the length here, since all the
blocks are the same size
+ * receyle bin.
* <p>
* @author Aaron Smuts
*/
public class BlockDiskElementDescriptor
- implements Serializable
+ implements Serializable, Externalizable
{
private static final long serialVersionUID = -1400659301208101411L;
@@ -81,5 +83,31 @@
}
buf.append( "]" );
return buf.toString();
+ }
+
+ /**
+ * Saves on reflection.
+ * <p>
+ * (non-Javadoc)
+ * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
+ */
+ public void readExternal( ObjectInput input )
+ throws IOException, ClassNotFoundException
+ {
+ this.key = (Serializable) input.readObject();
+ this.blocks = (int[]) input.readObject();
+ }
+
+ /**
+ * Saves on reflection.
+ * <p>
+ * (non-Javadoc)
+ * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
+ */
+ public void writeExternal( ObjectOutput output )
+ throws IOException
+ {
+ output.writeObject( this.key );
+ output.writeObject( this.blocks );
}
}
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
Sun Aug 27 11:20:07 2006
@@ -11,12 +11,15 @@
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
+import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.Serializable;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
@@ -46,6 +49,7 @@
private int maxKeySize;
+ // we need this so we can communicate free blocks to the data store when
keys fall off the LRU
private BlockDiskCache blockDiskCache;
private File rootDirectory;
@@ -72,11 +76,16 @@
if ( log.isInfoEnabled() )
{
- log.info( logCacheName + "Cache file root directory: " +
rootDirName );
+ log.info( logCacheName + "Cache file root directory [" +
rootDirName + "]" );
}
this.keyFile = new File( rootDirectory, fileName + ".key" );
+ if ( log.isInfoEnabled() )
+ {
+ log.info( logCacheName + "Key File [" +
this.keyFile.getAbsolutePath() + "]" );
+ }
+
if ( keyFile.length() > 0 )
{
loadKeys();
@@ -89,7 +98,8 @@
}
/**
- * Saves key file to disk. This converts the LRUMap to a HashMap for
deserialzation.
+ * Saves key file to disk. This gets the LRUMap entry set and write the
entries out one by one
+ * after putting them in a wrapper.
*/
protected void saveKeys()
{
@@ -99,34 +109,40 @@
int numKeys = keyHash.size();
if ( log.isInfoEnabled() )
{
- log.info( logCacheName + "Saving keys to: " + fileName + ",
key count: " + numKeys );
+ log.info( logCacheName + "Saving keys to [" +
this.keyFile.getAbsolutePath() + "], key count ["
+ + numKeys + "]" );
}
keyFile.delete();
- HashMap keys = new HashMap();
- keys.putAll( keyHash );
-
- if ( keys.size() > 0 )
+ keyFile = new File( rootDirectory, fileName + ".key" );
+ FileOutputStream fos = new FileOutputStream( keyFile );
+ BufferedOutputStream bos = new BufferedOutputStream( fos, 1024 );
+ ObjectOutputStream oos = new ObjectOutputStream( bos );
+ try
{
- FileOutputStream fos = new FileOutputStream( keyFile );
- BufferedOutputStream bos = new BufferedOutputStream( fos );
- ObjectOutputStream oos = new ObjectOutputStream( bos );
- try
- {
- oos.writeObject( keys );
- oos.flush();
- }
- finally
+ // don't need to synchronize, since the underlying collection
makes a copy
+ Iterator keyIt = keyHash.entrySet().iterator();
+ while ( keyIt.hasNext() )
{
- oos.close();
+ Map.Entry entry = (Map.Entry) keyIt.next();
+ BlockDiskElementDescriptor descriptor = new
BlockDiskElementDescriptor();
+ descriptor.setKey( (Serializable) entry.getKey() );
+ descriptor.setBlocks( (int[]) entry.getValue() );
+ // stream these out in the loop.
+ oos.writeObject( descriptor );
}
}
+ finally
+ {
+ oos.flush();
+ oos.close();
+ }
if ( log.isInfoEnabled() )
{
log.info( logCacheName + "Finished saving keys. It took " +
timer.getElapsedTimeString() + " to store "
- + numKeys + " keys." );
+ + numKeys + " keys. Key file length [" + keyFile.length()
+ "]" );
}
}
catch ( Exception e )
@@ -149,6 +165,14 @@
}
/**
+ * This is mainly used for testing. It leave the disk in tact, and just
clears memory.
+ */
+ protected void clearMemoryMap()
+ {
+ this.keyHash.clear();
+ }
+
+ /**
* Create the map for keys that contain the index position on disk.
*/
private void initKeyMap()
@@ -175,17 +199,17 @@
}
/**
- * Loads the keys from the .key file. The keys are stored in a HashMap on
disk. This is
- * converted into a LRUMap.
+ * Loads the keys from the .key file. The keys are stored individually on
disk. They are added
+ * one by one to an LRUMap..
* <p>
* @throws InterruptedException
*/
protected void loadKeys()
throws InterruptedException
{
- if ( log.isDebugEnabled() )
+ if ( log.isInfoEnabled() )
{
- log.debug( logCacheName + "Loading keys for " + keyFile.toString()
);
+ log.info( logCacheName + "Loading keys for " + keyFile.toString()
);
}
try
@@ -193,21 +217,32 @@
// create a key map to use.
initKeyMap();
- HashMap keys = null;
+ HashMap keys = new HashMap();
FileInputStream fis = new FileInputStream( keyFile );
BufferedInputStream bis = new BufferedInputStream( fis );
ObjectInputStream ois = new ObjectInputStream( bis );
try
{
- keys = (HashMap) ois.readObject();// 0, keyFile.length() );
+ while ( true )
+ {
+ BlockDiskElementDescriptor descriptor =
(BlockDiskElementDescriptor) ois.readObject();
+ if ( descriptor != null )
+ {
+ keys.put( descriptor.getKey(), descriptor.getBlocks()
);
+ }
+ }
+ }
+ catch ( EOFException eof )
+ {
+ // nothing
}
finally
{
ois.close();
}
- if ( keys != null )
+ if ( !keys.isEmpty() )
{
if ( log.isDebugEnabled() )
{
@@ -260,23 +295,23 @@
}
/**
- * gets the object fot he key.
+ * gets the object for the key.
* <p>
* @param key
* @return Object
*/
- public BlockDiskElementDescriptor get( Object key )
+ public int[] get( Object key )
{
- return (BlockDiskElementDescriptor) this.keyHash.get( key );
+ return (int[]) this.keyHash.get( key );
}
/**
- * Puts a BlockDiskElementDescriptor in the keyStore.
+ * Puts a int[] in the keyStore.
* <p>
* @param key
* @param value
*/
- public void put( Object key, BlockDiskElementDescriptor value )
+ public void put( Object key, int[] value )
{
this.keyHash.put( key, value );
}
@@ -287,9 +322,9 @@
* @param key
* @return BlockDiskElementDescriptor if it was present, else null
*/
- public BlockDiskElementDescriptor remove( Object key )
+ public int[] remove( Object key )
{
- return (BlockDiskElementDescriptor) this.keyHash.remove( key );
+ return (int[]) this.keyHash.remove( key );
}
/**
@@ -328,7 +363,7 @@
*/
protected void processRemovedLRU( Object key, Object value )
{
- blockDiskCache.freeBlocks( ( (BlockDiskElementDescriptor) value
).getBlocks() );
+ blockDiskCache.freeBlocks( (int[]) value );
if ( log.isDebugEnabled() )
{
log.debug( logCacheName + "Removing key: [" + key + "] from
key store." );
Modified: jakarta/jcs/trunk/src/java/org/apache/jcs/utils/struct/LRUMap.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/utils/struct/LRUMap.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
--- jakarta/jcs/trunk/src/java/org/apache/jcs/utils/struct/LRUMap.java
(original)
+++ jakarta/jcs/trunk/src/java/org/apache/jcs/utils/struct/LRUMap.java Sun Aug
27 11:20:07 2006
@@ -170,10 +170,9 @@
* <p>
* @see java.util.Map#entrySet()
*/
- public Set entrySet()
+ public synchronized Set entrySet()
{
// todo, we should return a defensive copy
- // this is not thread safe.
Set entries = map.entrySet();
Set unWrapped = new HashSet();
@@ -182,9 +181,7 @@
while ( it.hasNext() )
{
Entry pre = (Entry) it.next();
-
Entry post = new LRUMapEntry( pre.getKey(), (
(LRUElementDescriptor) pre.getValue() ).getPayload() );
-
unWrapped.add( post );
}
Added:
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheKeyStoreUnitTest.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheKeyStoreUnitTest.java?rev=437431&view=auto
==============================================================================
---
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheKeyStoreUnitTest.java
(added)
+++
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheKeyStoreUnitTest.java
Sun Aug 27 11:20:07 2006
@@ -0,0 +1,116 @@
+package org.apache.jcs.auxiliary.disk.block;
+
+/*
+ * Copyright 2001-2004 The Apache Software Foundation. Licensed under the
Apache License, Version
+ * 2.0 (the "License") you may not use this file except in compliance with the
License. You may
+ * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by
+ * applicable law or agreed to in writing, software distributed under the
License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied. See
+ * the License for the specific language governing permissions and limitations
under the License.
+ */
+
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for the keyStore.
+ * <p>
+ * @author Aaron Smuts
+ */
+public class BlockDiskCacheKeyStoreUnitTest
+ extends TestCase
+{
+ private String rootDirName = "target/test-sandbox/block";
+
+ /**
+ * Put a bunch of keys inthe key store and verify that they are present.
+ * <p>
+ * @throws Exception
+ */
+ public void testPutKeys()
+ throws Exception
+ {
+ // SETUP
+ String regionName = "testPutKeys";
+ int maxKeys = 1000;
+ int bytesPerBlock = 2000;
+
+ BlockDiskCacheAttributes attributes = new BlockDiskCacheAttributes();
+ attributes.setCacheName( regionName );
+ attributes.setDiskPath( rootDirName );
+ attributes.setMaxKeySize( maxKeys );
+ attributes.setBlockSizeBytes( bytesPerBlock );
+
+ BlockDiskCache blockDiskCache = new BlockDiskCache( attributes );
+
+ BlockDiskKeyStore keyStore = new BlockDiskKeyStore( attributes,
blockDiskCache );
+
+ // DO WORK
+ int numElements = 100;
+ for ( int i = 0; i < numElements; i++ )
+ {
+ keyStore.put( String.valueOf( i ), new int[i] );
+ }
+ System.out.println( "testPutKeys " + keyStore );
+
+ // VERIFY
+ assertEquals( "Wrong number of keys", numElements, keyStore.size() );
+ for ( int i = 0; i < numElements; i++ )
+ {
+ int[] result = keyStore.get( String.valueOf( i ) );
+ assertEquals( "Wrong array returned.", i, result.length );
+ }
+ }
+
+ /**
+ * Verify that we can load keys that we saved. Add a bunch. Save them.
Clear the memory keyhash.
+ * Load the keys. Verify.
+ * <p>
+ * @throws Exception
+ */
+ public void testSaveLoadKeys()
+ throws Exception
+ {
+ // SETUP
+ String regionName = "testSaveLoadKeys";
+ int maxKeys = 10000;
+ int bytesPerBlock = 2000;
+
+ BlockDiskCacheAttributes attributes = new BlockDiskCacheAttributes();
+ attributes.setCacheName( regionName );
+ attributes.setDiskPath( rootDirName );
+ attributes.setMaxKeySize( maxKeys );
+ attributes.setBlockSizeBytes( bytesPerBlock );
+
+ BlockDiskCache blockDiskCache = new BlockDiskCache( attributes );
+
+ BlockDiskKeyStore keyStore = new BlockDiskKeyStore( attributes,
blockDiskCache );
+
+ // DO WORK
+ int numElements = 1000;
+ Random random = new Random( 89 );
+ for ( int i = 0; i < numElements; i++ )
+ {
+ int blocks = random.nextInt( 10 );
+ keyStore.put( String.valueOf( i ), new int[blocks] );
+ }
+ System.out.println( "testSaveLoadKeys " + keyStore );
+
+ // VERIFY
+ assertEquals( "Wrong number of keys", numElements, keyStore.size() );
+
+ // DO WORK
+ keyStore.saveKeys();
+ keyStore.clearMemoryMap();
+
+ // VERIFY
+ assertEquals( "Wrong number of keys after clearing memory", 0,
keyStore.size() );
+
+ // DO WORK
+ keyStore.loadKeys();
+
+ // VERIFY
+ assertEquals( "Wrong number of keys after loading", numElements,
keyStore.size() );
+ }
+}
Modified:
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptorUnitTest.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptorUnitTest.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptorUnitTest.java
(original)
+++
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskElementDescriptorUnitTest.java
Sun Aug 27 11:20:07 2006
@@ -1,5 +1,14 @@
package org.apache.jcs.auxiliary.disk.block;
+/*
+ * Copyright 2001-2004 The Apache Software Foundation. Licensed under the
Apache License, Version
+ * 2.0 (the "License") you may not use this file except in compliance with the
License. You may
+ * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by
+ * applicable law or agreed to in writing, software distributed under the
License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied. See
+ * the License for the specific language governing permissions and limitations
under the License.
+ */
+
import junit.framework.TestCase;
/**
Modified:
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskUnitTest.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskUnitTest.java?rev=437431&r1=437430&r2=437431&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskUnitTest.java
(original)
+++
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskUnitTest.java
Sun Aug 27 11:20:07 2006
@@ -1,5 +1,14 @@
package org.apache.jcs.auxiliary.disk.block;
+/*
+ * Copyright 2001-2004 The Apache Software Foundation. Licensed under the
Apache License, Version
+ * 2.0 (the "License") you may not use this file except in compliance with the
License. You may
+ * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by
+ * applicable law or agreed to in writing, software distributed under the
License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied. See
+ * the License for the specific language governing permissions and limitations
under the License.
+ */
+
import java.io.File;
import junit.framework.TestCase;
|
