1 /***
2 * Copyright 2003-2010 Terracotta, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package net.sf.ehcache;
18
19 import static java.util.concurrent.TimeUnit.MILLISECONDS;
20 import static java.util.concurrent.TimeUnit.SECONDS;
21 import static net.sf.ehcache.util.RetryAssert.assertBy;
22 import static net.sf.ehcache.util.RetryAssert.sizeOnDiskOf;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertNotNull;
26 import static org.junit.Assert.assertNull;
27 import static org.junit.Assert.assertTrue;
28
29 import java.io.ByteArrayOutputStream;
30 import java.io.File;
31 import java.io.FileNotFoundException;
32 import java.io.FileOutputStream;
33 import java.io.IOException;
34 import java.io.ObjectOutputStream;
35 import java.io.RandomAccessFile;
36 import java.lang.reflect.Field;
37 import java.util.ArrayList;
38 import java.util.Date;
39 import java.util.List;
40 import java.util.Random;
41 import java.util.concurrent.Callable;
42
43 import net.sf.ehcache.config.CacheConfiguration;
44 import net.sf.ehcache.config.Configuration;
45 import net.sf.ehcache.config.DiskStoreConfiguration;
46 import net.sf.ehcache.config.MemoryUnit;
47 import net.sf.ehcache.store.FrontEndCacheTier;
48 import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
49 import net.sf.ehcache.store.Primitive;
50 import net.sf.ehcache.store.Store;
51 import net.sf.ehcache.store.disk.DiskStore;
52 import net.sf.ehcache.util.PropertyUtil;
53 import net.sf.ehcache.util.RetryAssert;
54
55 import org.hamcrest.core.Is;
56 import org.junit.After;
57 import org.junit.BeforeClass;
58 import org.junit.Test;
59 import org.slf4j.Logger;
60 import org.slf4j.LoggerFactory;
61
62 /***
63 * Test cases for the DiskStore.
64 *
65 * @author <a href="mailto:amurdoch@thoughtworks.com">Adam Murdoch</a>
66 * @author <a href="mailto:gluck@thoughtworks.com">Greg Luck</a>
67 * @version $Id: DiskStoreTest.html 13146 2011-08-01 17:12:39Z oletizi $
68 * <p/>
69 * total time 149 old i/o
70 * total time 133, 131, 130 nio
71 */
72 public class DiskStoreTest extends AbstractCacheTest {
73
74 private static final Logger LOG = LoggerFactory.getLogger(DiskStoreTest.class.getName());
75 private static final int ELEMENT_ON_DISK_SIZE = 1280;
76 private CacheManager manager2;
77
78 @BeforeClass
79 public static void enableHeapDump() {
80 setHeapDumpOnOutOfMemoryError(true);
81 }
82
83 /***
84 * teardown
85 */
86 @Override
87 @After
88 public void tearDown() throws Exception {
89 super.tearDown();
90 if (manager2 != null) {
91 manager2.shutdown();
92 }
93 deleteFile("persistentLongExpiryIntervalCache");
94 deleteFile("fileTest");
95 deleteFile("testPersistent");
96 deleteFile("testLoadPersistent");
97 deleteFile("testPersistentWithDelete");
98 }
99
100 /***
101 * Creates a store which is non-expiring so that we can check for
102 * size-related characteristics without elements being deleted under us.
103 */
104 private Store createNonExpiringDiskStore() {
105 Cache cache = new Cache("test/NonPersistent", 1, true, false, 2, 1, false, 1);
106 manager.addCache(cache);
107 return cache.getStore();
108 }
109
110 private Cache createDiskCache() {
111 Cache cache = new Cache(new CacheConfiguration().name("test/NonPersistent").maxEntriesLocalHeap(1).overflowToDisk(true)
112 .eternal(false).timeToLiveSeconds(2).timeToIdleSeconds(1).diskPersistent(false).diskExpiryThreadIntervalSeconds(1)
113 .diskSpoolBufferSizeMB(10));
114 manager.addCache(cache);
115 return cache;
116 }
117
118 private Store createDiskStore() {
119 return createDiskCache().getStore();
120 }
121
122 private Store createPersistentDiskStore(String cacheName) {
123 Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
124 manager.addCache(cache);
125 return cache.getStore();
126 }
127
128 private Store createAutoPersistentDiskStore(String cacheName) {
129 Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
130 manager2 = new CacheManager();
131
132 manager2.addCache(cache);
133 return cache.getStore();
134 }
135
136 private Store createPersistentDiskStoreFromCacheManager() {
137 Cache cache = manager.getCache("persistentLongExpiryIntervalCache");
138 return cache.getStore();
139 }
140
141 private Store createCapacityLimitedDiskStore() {
142 Cache cache = new Cache("test/CapacityLimited", 1, MemoryStoreEvictionPolicy.LRU, true, null, true,
143 0, 0, false, 600, null, null, 50);
144 manager.addCache(cache);
145 return cache.getStore();
146 }
147
148 private Cache createStripedDiskCache(int stripes) {
149 CacheConfiguration config = new CacheConfiguration("test/NonPersistentStriped_" + stripes, 10000).overflowToDisk(true)
150 .eternal(false).timeToLiveSeconds(2).timeToIdleSeconds(1).diskPersistent(false).diskExpiryThreadIntervalSeconds(1)
151 .diskAccessStripes(stripes).diskSpoolBufferSizeMB(10);
152 Cache cache = new Cache(config);
153 manager.addCache(cache);
154 return cache;
155 }
156
157 /***
158 * Test to help debug DiskStore test
159 */
160 @Test
161 public void testNothing() {
162
163 }
164
165 /***
166 * Tests that a file is created with the right size after puts, and that the file is
167 * deleted on disposal
168 */
169 @Test
170 public void testNonPersistentStore() throws Exception {
171 DiskStore diskStore = getDiskStore(createNonExpiringDiskStore());
172 File dataFile = diskStore.getDataFile();
173
174
175 for (int i = 0; i < 101; i++) {
176 byte[] data = new byte[1024];
177 diskStore.put(new Element("key" + (i + 100), data));
178 }
179 waitShorter();
180 assertEquals(ELEMENT_ON_DISK_SIZE * 100 + 1280, diskStore.getOnDiskSizeInBytes());
181
182 assertEquals(101, diskStore.getOnDiskSize());
183 assertEquals(101, diskStore.getSize());
184 diskStore.dispose();
185 Thread.sleep(1);
186 assertFalse("File exists", dataFile.exists());
187 }
188
189
190 /***
191 * Tests the auto generated directories are deleted
192 */
193 @Test
194 public void testDeleteAutoGenerated() {
195 manager2 = new CacheManager();
196 String diskPath = manager2.getDiskStorePath();
197 File dataDirectory = new File(diskPath);
198 assertTrue(dataDirectory.exists());
199 assertTrue(dataDirectory.isDirectory());
200 manager2.shutdown();
201
202 assertTrue(!dataDirectory.exists());
203 assertTrue(!dataDirectory.isDirectory());
204 }
205
206
207 /***
208 * Tests that the Disk Store can be changed
209 */
210 @Test
211 public void testSetDiskStorePath() throws Exception {
212 Cache cache = new Cache("testChangePath", 10000, true, false, 5, 1, true, 600);
213 manager2 = new CacheManager();
214 cache.setDiskStorePath(System.getProperty("java.io.tmpdir") + File.separator + "changedDiskStorePath");
215 manager2.addCache(cache);
216 DiskStore diskStore = getDiskStore(cache.getStore());
217 File dataFile = diskStore.getDataFile();
218 assertTrue("File exists", dataFile.exists());
219 }
220
221 /***
222 * Tests that a file is created with the right size after puts, and that the file is not
223 * deleted on disposal
224 * <p/>
225 * This test uses a preconfigured cache from the test cache.xml. Note that teardown causes
226 * an exception because the disk store is being shut down twice.
227 */
228 @Test
229 public void testPersistentStore() throws Exception {
230
231 DiskStore store = getDiskStore(createPersistentDiskStoreFromCacheManager());
232 store.removeAll();
233
234 File dataFile = store.getDataFile();
235
236 for (int i = 0; i < 100; i++) {
237 byte[] data = new byte[1024];
238 store.put(new Element("key" + (i + 100), data));
239 }
240 waitShorter();
241 assertEquals(100, store.getSize());
242 store.dispose();
243
244 assertTrue("File exists", dataFile.exists());
245 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
246 }
247
248 /***
249 * An integration test, at the CacheManager level, to make sure persistence works
250 */
251 @Test
252 public void testPersistentStoreFromCacheManager() throws IOException, InterruptedException, CacheException {
253
254 CacheManager manager = new CacheManager(AbstractCacheTest.TEST_CONFIG_DIR + "ehcache-disk.xml");
255 Ehcache cache = manager.getCache("persistentLongExpiryIntervalCache");
256
257 LOG.info("DiskStore path: {}", manager.getDiskStorePath());
258
259 for (int i = 0; i < 100; i++) {
260 byte[] data = new byte[1024];
261 cache.put(new Element("key" + (i + 100), data));
262 }
263 assertEquals(100, cache.getSize());
264
265 manager.shutdown();
266
267 manager = new CacheManager(AbstractCacheTest.TEST_CONFIG_DIR + "ehcache-disk.xml");
268 cache = manager.getCache("persistentLongExpiryIntervalCache");
269 assertEquals(100, cache.getSize());
270
271 manager.shutdown();
272
273 }
274
275 /***
276 * An integration test, at the CacheManager level, to make sure persistence works
277 * This test checks the config where a cache is not configured overflow to disk, but is disk persistent.
278 * It should work by putting elements in the DiskStore initially and then loading them into memory as they
279 * are called.
280 */
281 @Test
282 public void testPersistentNonOverflowToDiskStoreFromCacheManager() throws IOException, InterruptedException, CacheException {
283
284 {
285 CacheManager manager = new CacheManager(AbstractCacheTest.TEST_CONFIG_DIR + "ehcache-disk.xml");
286 final Ehcache cache = manager.getCache("persistentLongExpiryIntervalNonOverflowCache");
287
288 for (int i = 0; i < 100; i++) {
289 byte[] data = new byte[1024];
290 cache.put(new Element("key" + (i + 100), data));
291 }
292 assertEquals(100, cache.getSize());
293
294 manager.shutdown();
295 }
296
297 {
298 CacheManager manager = new CacheManager(AbstractCacheTest.TEST_CONFIG_DIR + "ehcache-disk.xml");
299 final Ehcache cache = manager.getCache("persistentLongExpiryIntervalNonOverflowCache");
300
301
302 RetryAssert.assertBy(500, MILLISECONDS, new Callable<Integer>() {
303 public Integer call() throws Exception {
304 return cache.getSize();
305 }
306 }, Is.is(100));
307
308 assertEquals(100, cache.getDiskStoreSize());
309 assertEquals(100, cache.getKeysNoDuplicateCheck().size());
310 assertEquals(100, cache.getKeys().size());
311 assertEquals(100, cache.getKeysWithExpiryCheck().size());
312
313
314 assertNotNull(cache.get("key100"));
315 assertNotNull(cache.getQuiet("key100"));
316 cache.remove("key100");
317 assertNull(cache.get("key100"));
318 assertNull(cache.getQuiet("key100"));
319 cache.removeAll();
320 assertEquals(0, cache.getSize());
321 assertEquals(0, cache.getDiskStoreSize());
322
323 manager.shutdown();
324 }
325 }
326
327 /***
328 * Tests that we can save and load a persistent store in a repeatable way
329 */
330 @Test
331 public void testLoadPersistentStore() throws Exception {
332
333 String cacheName = "testLoadPersistent";
334 Store store = createPersistentDiskStore(cacheName);
335 store.removeAll();
336 waitShorter();
337
338
339 for (int i = 0; i < 100; i++) {
340 byte[] data = new byte[1024];
341 store.put(new Element("key" + (i + 100), data));
342 }
343 store.flush();
344 waitShorter();
345 assertEquals(ELEMENT_ON_DISK_SIZE * 100, store.getOnDiskSizeInBytes());
346 assertEquals(100, store.getSize());
347 manager.removeCache(cacheName);
348 Thread.sleep(3000);
349
350 for (int i = 0; i < 10; i++) {
351 store = getDiskStore(createPersistentDiskStore(cacheName));
352 File dataFile = ((DiskStore) store).getDataFile();
353 assertTrue("File exists", dataFile.exists());
354 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
355 assertEquals(100, store.getSize());
356
357 manager.removeCache(cacheName);
358
359 assertTrue("File exists", dataFile.exists());
360 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
361 }
362 }
363
364 /***
365 * Any disk store with an auto generated random directory should not be able to be loaded.
366 */
367 @Test
368 public void testCannotLoadPersistentStoreWithAutoDir() throws Exception {
369
370 String cacheName = "testPersistent";
371 Store diskStore = createAutoPersistentDiskStore(cacheName);
372 diskStore.removeAll();
373
374
375 for (int i = 0; i < 100; i++) {
376 byte[] data = new byte[1024];
377 diskStore.put(new Element("key" + (i + 100), data));
378 }
379 waitLonger();
380 assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
381 assertEquals(100, diskStore.getSize());
382 manager2.removeCache(cacheName);
383 Thread.sleep(1000);
384
385 Cache cache = new Cache(cacheName, 10000, true, false, 5, 1, true, 600);
386 manager2.addCache(cache);
387
388 File dataFile = getDiskStore(diskStore).getDataFile();
389 assertTrue("File exists", dataFile.exists());
390 assertEquals(0, dataFile.length());
391 assertEquals(0, cache.getSize());
392 manager.removeCache(cacheName);
393 assertTrue("File exists", dataFile.exists());
394 assertEquals(0, dataFile.length());
395 }
396
397 /***
398 * Tests that we can save and load a persistent store in a repeatable way,
399 * and delete and add data.
400 */
401 @Test
402 public void testLoadPersistentStoreWithDelete() throws Exception {
403
404 String cacheName = "testPersistentWithDelete";
405 Store diskStore = createPersistentDiskStore(cacheName);
406 diskStore.removeAll();
407
408
409 for (int i = 0; i < 100; i++) {
410 byte[] data = new byte[1024];
411 diskStore.put(new Element("key" + (i + 100), data));
412 }
413 waitShorter();
414 assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
415 assertEquals(100, diskStore.getSize());
416 manager.removeCache(cacheName);
417
418 diskStore = getDiskStore(createPersistentDiskStore(cacheName));
419 File dataFile = ((DiskStore) diskStore).getDataFile();
420 assertTrue("File exists", dataFile.exists());
421 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
422 assertEquals(100, diskStore.getSize());
423
424 diskStore.remove("key100");
425 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
426 assertEquals(99, diskStore.getSize());
427
428 manager.removeCache(cacheName);
429
430 diskStore = createPersistentDiskStore(cacheName);
431 assertTrue("File exists", dataFile.exists());
432 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
433 assertEquals(99, diskStore.getSize());
434
435 diskStore.put(new Element("key100", new byte[1024]));
436 diskStore.flush();
437 waitShorter();
438 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
439 assertEquals(100, diskStore.getSize());
440 }
441
442 /***
443 * Tests that we can load a store after the index has been corrupted
444 */
445 @Test
446 public void testLoadPersistentStoreAfterCorruption() throws Exception {
447
448 String cacheName = "testPersistent";
449 DiskStore diskStore = getDiskStore(createPersistentDiskStore(cacheName));
450 diskStore.removeAll();
451
452
453 for (int i = 0; i < 100; i++) {
454 byte[] data = new byte[1024];
455 diskStore.put(new Element("key" + (i + 100), data));
456 }
457 assertEquals(100, diskStore.getSize());
458 manager.removeCache(cacheName);
459
460 File dataFile = diskStore.getDataFile();
461 assertTrue(dataFile.length() >= 100 * ELEMENT_ON_DISK_SIZE);
462
463 File indexFile = diskStore.getIndexFile();
464 FileOutputStream fout = new FileOutputStream(indexFile);
465
466 fout.write(new byte[]{'q', 'w', 'e', 'r', 't', 'y'});
467 fout.close();
468
469 diskStore = getDiskStore(createPersistentDiskStore(cacheName));
470 assertTrue("File exists", dataFile.exists());
471
472
473 assertEquals("Data file was not recreated", 0, dataFile.length());
474 assertEquals(0, diskStore.getSize());
475 }
476
477 /***
478 * Tests that we can save and load a persistent store in a repeatable way,
479 * and delete and add data.
480 */
481 @Test
482 public void testFreeSpaceBehaviour() throws Exception {
483
484 String cacheName = "testPersistent";
485 Store diskStore = createPersistentDiskStore(cacheName);
486 diskStore.removeAll();
487
488 byte[] data = new byte[1024];
489 for (int i = 0; i < 100; i++) {
490 diskStore.put(new Element("key" + (i + 100), data));
491 }
492 waitShorter();
493 assertEquals(ELEMENT_ON_DISK_SIZE * 100, diskStore.getOnDiskSizeInBytes());
494 assertEquals(100, diskStore.getSize());
495 manager.removeCache(cacheName);
496
497 diskStore = createPersistentDiskStore(cacheName);
498 File dataFile = getDiskStore(diskStore).getDataFile();
499 assertTrue("File exists", dataFile.exists());
500 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
501 assertEquals(100, diskStore.getSize());
502
503 diskStore.remove("key100");
504 diskStore.remove("key101");
505 diskStore.remove("key102");
506 diskStore.remove("key103");
507 diskStore.remove("key104");
508
509 diskStore.put(new Element("key100", data));
510 diskStore.put(new Element("key101", data));
511 waitShorter();
512
513
514 assertEquals(100 * ELEMENT_ON_DISK_SIZE, dataFile.length());
515 assertEquals(97, diskStore.getSize());
516
517 diskStore.put(new Element("key102", data));
518 diskStore.put(new Element("key103", data));
519 diskStore.put(new Element("key104", data));
520 diskStore.put(new Element("key201", data));
521 diskStore.put(new Element("key202", data));
522 waitShorter();
523 assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
524 assertEquals(102, diskStore.getSize());
525 manager.removeCache(cacheName);
526 assertTrue("File exists", dataFile.exists());
527 assertEquals(102 * ELEMENT_ON_DISK_SIZE, dataFile.length());
528 }
529
530 /***
531 * Tests looking up an entry that does not exist.
532 */
533 @Test
534 public void testGetUnknownThenKnown() throws Exception {
535 final Store diskStore = createDiskStore();
536 assertNull(diskStore.get("key1"));
537 diskStore.put(new Element("key1", "value"));
538 diskStore.put(new Element("key2", "value"));
539 assertNotNull(diskStore.get("key1"));
540 assertNotNull(diskStore.get("key2"));
541 }
542
543 /***
544 * Tests looking up an entry that does not exist.
545 */
546 @Test
547 public void testGetQuietUnknownThenKnown() throws Exception {
548 final Store diskStore = createDiskStore();
549 assertNull(diskStore.getQuiet("key1"));
550 diskStore.put(new Element("key1", "value"));
551 diskStore.put(new Element("key2", "value"));
552 assertNotNull(diskStore.getQuiet("key1"));
553 assertNotNull(diskStore.getQuiet("key2"));
554 }
555
556 /***
557 * Tests adding an entry.
558 */
559 @Test
560 public void testPut() throws Exception {
561 final Store diskStore = createDiskStore();
562
563
564 assertEquals(0, diskStore.getSize());
565 assertNull(diskStore.get("key"));
566
567
568 final String value = "value";
569 diskStore.put(new Element("key1", value));
570 diskStore.put(new Element("key2", value));
571
572 Thread.sleep(1000);
573
574
575 assertEquals(2, diskStore.getSize());
576 assertEquals(2, diskStore.getOnDiskSize());
577
578 Element element1 = diskStore.get("key1");
579 Element element2 = diskStore.get("key2");
580 assertNotNull(element1);
581 assertNotNull(element2);
582 assertEquals(value, element1.getObjectValue());
583 assertEquals(value, element2.getObjectValue());
584 }
585
586
587 /***
588 *
589 */
590 @Test
591 public void testLFUEvictionFromDiskStore() throws IOException, InterruptedException {
592 Cache cache = new Cache("testNonPersistent", 1, MemoryStoreEvictionPolicy.LFU, true,
593 null, false, 2000, 1000, false, 1, null, null, 10);
594 manager.addCache(cache);
595 final Store store = cache.getStore();
596
597 Element element;
598
599 assertEquals(0, store.getSize());
600
601 for (int i = 0; i < 10; i++) {
602 cache.put(new Element("key" + i, "value" + i));
603 assertBy(1, SECONDS, sizeOnDiskOf(store), Is.is(i + 1));
604 if (i > 0) {
605 cache.get("key" + i);
606 cache.get("key" + i);
607 cache.get("key" + i);
608 cache.get("key" + i);
609 }
610 }
611
612
613 assertBy(1, SECONDS, sizeOnDiskOf(store), Is.is(10));
614
615 element = new Element("keyNew", "valueNew");
616 store.put(element);
617
618
619 Thread.sleep(220);
620 assertEquals(10, store.getOnDiskSize());
621
622 assertNotNull(store.get(element.getObjectKey()));
623
624 assertNull(store.get("key0"));
625
626 for (int i = 0; i < 2000; i++) {
627 store.put(new Element("" + i, new Date()));
628 }
629
630 waitLonger();
631
632 assertBy(1, SECONDS, sizeOnDiskOf(store), Is.is(10));
633 }
634
635 /***
636 * Tests the loading of classes
637 */
638 @Test
639 public void testClassloading() throws Exception {
640 final Store diskStore = createDiskStore();
641
642 Long value = Long.valueOf(123L);
643 diskStore.put(new Element("key1", value));
644 diskStore.put(new Element("key2", value));
645 Thread.sleep(1000);
646 Element element1 = diskStore.get("key1");
647 Element element2 = diskStore.get("key2");
648 assertEquals(value, element1.getObjectValue());
649 assertEquals(value, element2.getObjectValue());
650
651
652 Primitive primitive = new Primitive();
653 primitive.integerPrimitive = 123;
654 primitive.longPrimitive = 456L;
655 primitive.bytePrimitive = "a".getBytes()[0];
656 primitive.charPrimitive = 'B';
657 primitive.booleanPrimitive = false;
658
659
660 ByteArrayOutputStream outstr = new ByteArrayOutputStream();
661 ObjectOutputStream objstr = new ObjectOutputStream(outstr);
662 objstr.writeObject(new Element("key", value));
663 objstr.close();
664
665
666 diskStore.put(new Element("primitive1", primitive));
667 diskStore.put(new Element("primitive2", primitive));
668 Thread.sleep(1000);
669 Element primitive1 = diskStore.get("primitive1");
670 Element primitive2 = diskStore.get("primitive2");
671 assertEquals(primitive, primitive1.getObjectValue());
672 assertEquals(primitive, primitive2.getObjectValue());
673 }
674
675
676 /***
677 * Tests adding an entry and waiting for it to be written.
678 */
679 @Test
680 public void testPutSlow() throws Exception {
681 final DiskStore diskStore = getDiskStore(createDiskStore());
682
683
684 assertEquals(0, diskStore.getSize());
685 assertNull(diskStore.get("key"));
686
687
688 final String value = "value";
689 diskStore.put(new Element("key1", value));
690 diskStore.put(new Element("key2", value));
691
692
693 waitShorter();
694
695
696 assertEquals(2, diskStore.getOnDiskSize());
697 assertEquals(2, diskStore.getSize());
698
699 Element element1 = diskStore.get("key1");
700 Element element2 = diskStore.get("key2");
701 assertNotNull(element1);
702 assertNotNull(element2);
703 assertEquals(value, element1.getObjectValue());
704 assertEquals(value, element2.getObjectValue());
705 }
706
707 /***
708 * Tests removing an entry.
709 */
710 @Test
711 public void testRemove() throws Exception {
712 final DiskStore diskStore = getDiskStore(createDiskStore());
713
714
715 final String value = "value";
716 diskStore.put(new Element("key1", value));
717 diskStore.put(new Element("key2", value));
718
719 Thread.sleep(1000);
720
721
722 assertEquals(2, diskStore.getOnDiskSize());
723 assertEquals(2, diskStore.getSize());
724
725 assertNotNull(diskStore.get("key1"));
726 assertNotNull(diskStore.get("key2"));
727
728
729 diskStore.remove("key1");
730 diskStore.remove("key2");
731
732 waitShorter();
733
734
735 assertEquals(0, diskStore.getOnDiskSize());
736 assertEquals(0, diskStore.getSize());
737 assertNull(diskStore.get("key1"));
738 assertNull(diskStore.get("key2"));
739 }
740
741 @Test
742 public void testPersistentChangingPoolSizeBetweenRestarts() throws Exception {
743 String diskStorePath = System.getProperty("java.io.tmpdir") + File.separatorChar + "testPersistentChangingPoolSizeBetweenRestarts";
744 manager = new CacheManager(
745 new Configuration()
746 .diskStore(new DiskStoreConfiguration().path(diskStorePath))
747
748 );
749
750 manager.addCache(new Cache(
751 new CacheConfiguration()
752 .name("persistentCache")
753 .overflowToDisk(true)
754 .diskPersistent(true)
755 )
756 );
757 Cache cache = manager.getCache("persistentCache");
758
759 for (int i = 0; i < 500; i++) {
760 cache.put(new Element(i, new byte[1024]));
761 }
762
763 assertEquals(500, cache.getSize());
764
765 manager.shutdown();
766
767 manager = new CacheManager(
768 new Configuration()
769 .diskStore(new DiskStoreConfiguration().path(diskStorePath)
770 )
771 );
772
773 manager.addCache(new Cache(
774 new CacheConfiguration()
775 .name("persistentCache")
776 .overflowToDisk(true)
777 .diskPersistent(true)
778 .maxBytesLocalDisk(100, MemoryUnit.KILOBYTES)
779 )
780 );
781 cache = manager.getCache("persistentCache");
782
783 assertTrue(cache.getSize() <= 100);
784
785 manager.shutdown();
786 }
787
788
789 /***
790 * Tests removing an entry, after it has been written
791 */
792 @Test
793 public void testRemoveSlow() throws Exception {
794 final DiskStore diskStore = getDiskStore(createDiskStore());
795
796
797 final String value = "value";
798 diskStore.put(new Element("key1", value));
799 diskStore.put(new Element("key2", value));
800
801
802 waitShorter();
803
804
805 assertEquals(2, diskStore.getOnDiskSize());
806 assertEquals(2, diskStore.getSize());
807
808
809 diskStore.remove("key1");
810 diskStore.remove("key2");
811
812
813 assertEquals(0, diskStore.getSize());
814 assertEquals(0, diskStore.getOnDiskSize());
815 assertNull(diskStore.get("key1"));
816 assertNull(diskStore.get("key2"));
817 }
818
819 /***
820 * Tests removing all the entries.
821 */
822 @Test
823 public void testRemoveAll() throws Exception {
824 final DiskStore diskStore = getDiskStore(createDiskStore());
825
826
827 final String value = "value";
828 diskStore.put(new Element("key1", value));
829 diskStore.put(new Element("key2", value));
830
831
832 assertNotNull(diskStore.get("key1"));
833 assertNotNull(diskStore.get("key2"));
834
835
836 diskStore.removeAll();
837
838
839 assertEquals(0, diskStore.getSize());
840 assertEquals(0, diskStore.getOnDiskSize());
841 assertNull(diskStore.get("key1"));
842 assertNull(diskStore.get("key2"));
843 }
844
845 /***
846 * Tests removing all the entries, after they have been written to disk.
847 */
848 @Test
849 public void testRemoveAllSlow() throws Exception {
850 final DiskStore diskStore = getDiskStore(createDiskStore());
851
852
853 final String value = "value";
854 diskStore.put(new Element("key1", value));
855 diskStore.put(new Element("key2", value));
856
857
858 waitShorter();
859
860
861 diskStore.removeAll();
862
863
864 assertEquals(0, diskStore.getSize());
865 assertEquals(0, diskStore.getOnDiskSize());
866 assertNull(diskStore.get("key1"));
867 assertNull(diskStore.get("key2"));
868 }
869
870 /***
871 * Tests bulk load.
872 */
873 @Test
874 public void testBulkLoad() throws Exception {
875 final Store diskStore = createDiskStore();
876
877 final Random random = new Random();
878
879
880 for (int i = 0; i < 500; i++) {
881
882 final String key = "key" + i;
883 final String value = "This is a value" + random.nextInt(1000);
884
885
886 Element element = new Element(key, value);
887 diskStore.put(element);
888 element = diskStore.get(key);
889 assertNotNull(element);
890
891
892 Thread.sleep(2);
893
894
895 diskStore.remove(key);
896 element = diskStore.get(key);
897 assertNull(element);
898
899 element = new Element(key, value);
900 diskStore.put(element);
901 element = diskStore.get(key);
902 assertNotNull(element);
903
904
905 Thread.sleep(2);
906 }
907 }
908
909 /***
910 * Tests for element expiry.
911 */
912 @Test
913 public void testExpiry() throws Exception {
914
915 final DiskStore diskStore = getDiskStore(createDiskStore());
916
917
918 diskStore.put(new Element("key1", "value", false, 1, 1));
919 diskStore.put(new Element("key2", "value", false, 1, 1));
920
921
922 Thread.sleep(200);
923
924 assertEquals(2, diskStore.getSize());
925 assertEquals(2, diskStore.getOnDiskSize());
926
927
928 Thread.sleep(3000);
929
930 Element e1 = diskStore.get("key1");
931 Element e2 = diskStore.get("key2");
932
933 assertNull(e2);
934 assertNull(e1);
935 }
936
937 /***
938 * Checks that the expiry thread runs and expires elements which has the effect
939 * of preventing the disk store from continously growing.
940 * Ran for 6 hours through 10000 outer loops. No memory use increase.
941 * Using a key of "key" + i * outer) you get early slots that cannot be reused. The DiskStore
942 * actual size therefore starts at 133890 and ends at 616830. There is quite a lot of space
943 * that cannot be used because of fragmentation. Question? Should an effort be made to coalesce
944 * fragmented space? Unlikely in production to get contiguous fragments as in the first form
945 * of this test.
946 * <p/>
947 * Using a key of Integer.valueOf(i * outer) the size stays constant at 140800.
948 *
949 * @throws InterruptedException
950 */
951 @Test
952 public void testExpiryWithSize() throws InterruptedException {
953 Store diskStore = createDiskStore();
954 diskStore.removeAll();
955
956 byte[] data = new byte[1024];
957 for (int outer = 1; outer <= 10; outer++) {
958 for (int i = 0; i < 101; i++) {
959 Element element = new Element(Integer.valueOf(i * outer), data);
960 element.setTimeToLive(1);
961 diskStore.put(element);
962 }
963 waitLonger();
964 int predictedSize = (ELEMENT_ON_DISK_SIZE + 82) * 100;
965 long actualSize = diskStore.getOnDiskSizeInBytes();
966 LOG.info("Predicted Size: " + predictedSize + " Actual Size: " + actualSize);
967 assertTrue(actualSize <= predictedSize);
968 LOG.info("Memory Use: " + measureMemoryUse());
969 }
970 }
971
972 /***
973 * Waits for all spooled elements to be written to disk.
974 */
975 private static void waitShorter() throws InterruptedException {
976 Thread.sleep((long) (300 + 100 * getSpeedAdjustmentFactor()));
977 }
978
979 private static float getSpeedAdjustmentFactor() {
980 final String speedAdjustmentFactorString = PropertyUtil.extractAndLogProperty("net.sf.ehcache.speedAdjustmentFactor", System.getProperties());
981 if (speedAdjustmentFactorString != null) {
982 return Float.parseFloat(speedAdjustmentFactorString);
983 } else {
984 return 1;
985 }
986 }
987
988
989 /***
990 * Waits for all spooled elements to be written to disk.
991 */
992 private static void waitLonger() throws InterruptedException {
993 Thread.sleep((long) (300 + 500 * getSpeedAdjustmentFactor()));
994 }
995
996
997 /***
998 * Multi-thread read-only test. Will fail on memory constrained VMs
999 */
1000 @Test
1001 public void testReadOnlyMultipleThreads() throws Exception {
1002 final Store diskStore = createNonExpiringDiskStore();
1003
1004
1005 diskStore.put(new Element("key0", "value"));
1006 diskStore.put(new Element("key1", "value"));
1007 diskStore.put(new Element("key2", "value"));
1008 diskStore.put(new Element("key3", "value"));
1009
1010
1011 waitShorter();
1012
1013
1014 final List executables = new ArrayList();
1015 for (int i = 0; i < 20; i++) {
1016 final String key = "key" + (i % 4);
1017 final Executable executable = new Executable() {
1018 public void execute() throws Exception {
1019 final Element element = diskStore.get(key);
1020 assertNotNull(element);
1021 assertEquals("value", element.getObjectValue());
1022 }
1023 };
1024 executables.add(executable);
1025 }
1026 runThreads(executables);
1027 }
1028
1029 /***
1030 * Multi-thread concurrent read remove test.
1031 */
1032 @Test
1033 public void testReadRemoveMultipleThreads() throws Exception {
1034 final Random random = new Random();
1035 final Cache diskCache = createDiskCache();
1036
1037 diskCache.put(new Element("key", "value"));
1038
1039
1040 final List executables = new ArrayList();
1041 for (int i = 0; i < 5; i++) {
1042 final Executable executable = new Executable() {
1043 public void execute() throws Exception {
1044 for (int i = 0; i < 100; i++) {
1045 diskCache.put(new Element("key" + random.nextInt(100), "value"));
1046 }
1047 }
1048 };
1049 executables.add(executable);
1050 }
1051 for (int i = 0; i < 5; i++) {
1052 final Executable executable = new Executable() {
1053 public void execute() throws Exception {
1054 for (int i = 0; i < 100; i++) {
1055 diskCache.remove("key" + random.nextInt(100));
1056 }
1057 }
1058 };
1059 executables.add(executable);
1060 }
1061
1062 runThreads(executables);
1063 }
1064
1065 @Test
1066 public void testReadRemoveMultipleThreadsMultipleStripes() throws Exception {
1067 for (int stripes = 0; stripes < 10; stripes++) {
1068 final Random random = new Random();
1069 final Cache cache = createStripedDiskCache(stripes);
1070 try {
1071 cache.put(new Element("key", "value"));
1072
1073
1074 final List executables = new ArrayList();
1075 for (int i = 0; i < 5; i++) {
1076 final Executable executable = new Executable() {
1077 public void execute() throws Exception {
1078 for (int i = 0; i < 100; i++) {
1079 cache.put(new Element("key" + random.nextInt(100), "value"));
1080 }
1081 }
1082 };
1083 executables.add(executable);
1084 }
1085 for (int i = 0; i < 5; i++) {
1086 final Executable executable = new Executable() {
1087 public void execute() throws Exception {
1088 for (int i = 0; i < 100; i++) {
1089 cache.remove("key" + random.nextInt(100));
1090 }
1091 }
1092 };
1093 executables.add(executable);
1094 }
1095
1096 runThreads(executables);
1097 } finally {
1098 manager.removeCache(cache.getName());
1099 }
1100 }
1101 }
1102
1103 /***
1104 * Tests how data is written to a random access file.
1105 * <p/>
1106 * It makes sure that bytes are immediately written to disk after a write.
1107 */
1108 @Test
1109 public void testWriteToFile() throws IOException {
1110
1111 String dataFileName = "fileTest";
1112 RandomAccessFile file = getRandomAccessFile(dataFileName);
1113
1114
1115 byte[] buffer = new byte[1024];
1116 for (int i = 0; i < 100; i++) {
1117 file.write(buffer);
1118 }
1119
1120 assertEquals(1024 * 100, file.length());
1121
1122 }
1123
1124 private RandomAccessFile getRandomAccessFile(String name) throws FileNotFoundException {
1125 String diskPath = System.getProperty("java.io.tmpdir");
1126 final File diskDir = new File(diskPath);
1127 File dataFile = new File(diskDir, name + ".data");
1128 return new RandomAccessFile(dataFile, "rw");
1129 }
1130
1131
1132 /***
1133 * This test is designed to be used with a profiler to explore the ways in which DiskStore
1134 * uses memory. It does not do much on its own.
1135 */
1136 @Test
1137 public void testOutOfMemoryErrorOnOverflowToDisk() throws Exception {
1138
1139
1140 Cache cache = new Cache("test", 1000, MemoryStoreEvictionPolicy.LRU, true, null, false, 500, 500, false, 1, null);
1141 manager.addCache(cache);
1142 int i = 0;
1143
1144 Random random = new Random();
1145 for (; i < 5500; i++) {
1146 byte[] bytes = new byte[10000];
1147 random.nextBytes(bytes);
1148 cache.put(new Element("" + i, bytes));
1149 }
1150 LOG.info("Elements written: " + i);
1151
1152 }
1153
1154 /***
1155 * Java is not consistent with trailing file separators, believe it or not!
1156 * http://www.rationalpi.com/blog/replyToComment.action?entry=1146628709626&comment=1155660875090
1157 * Can we fix c:\temp//greg?
1158 */
1159 @Test
1160 public void testWindowsAndSolarisTempDirProblem() throws InterruptedException {
1161
1162 String originalPath = "c:" + File.separator + "temp" + File.separator + File.separator + "greg";
1163
1164 String translatedPath = new DiskStoreConfiguration().path(originalPath).getPath();
1165 assertEquals("c:" + File.separator + "temp" + File.separator + "greg", translatedPath);
1166
1167 translatedPath = new DiskStoreConfiguration().path(translatedPath).getPath();
1168 assertEquals("c:" + File.separator + "temp" + File.separator + "greg", translatedPath);
1169
1170 Thread.sleep(500);
1171 }
1172
1173 @Test
1174 public void testShrinkingAndGrowingDiskStore() throws Exception {
1175 Store diskBackedMemoryStore = createCapacityLimitedDiskStore();
1176 DiskStore store = getDiskStore(diskBackedMemoryStore);
1177
1178 int i = 0;
1179 store.put(new Element(Integer.valueOf(i++), new byte[100]));
1180 while (true) {
1181 int beforeSize = store.getOnDiskSize();
1182 store.put(new Element(Integer.valueOf(i++), new byte[100]));
1183 MILLISECONDS.sleep(500);
1184 int afterSize = store.getOnDiskSize();
1185 LOG.info(beforeSize + " ==> " + afterSize);
1186 if (afterSize <= beforeSize) {
1187 LOG.info("Hit Threshold : Terminating");
1188 break;
1189 }
1190 }
1191
1192 LOG.info("Wait For Spool Thread To Finish");
1193 SECONDS.sleep(2);
1194
1195 final int initialSize = store.getOnDiskSize();
1196 final int shrinkSize = initialSize / 2;
1197 store.changeDiskCapacity(shrinkSize);
1198 LOG.info("Resized : " + initialSize + " ==> " + shrinkSize);
1199
1200 LOG.info("Wait For Spool Thread To Finish");
1201 SECONDS.sleep(2);
1202
1203 for (; ; i++) {
1204 int beforeSize = store.getOnDiskSize();
1205 store.put(new Element(Integer.valueOf(i), new byte[100]));
1206 MILLISECONDS.sleep(500);
1207 int afterSize = store.getOnDiskSize();
1208 LOG.info(beforeSize + " ==> " + afterSize);
1209 if (afterSize >= beforeSize && afterSize <= shrinkSize * 1.1) {
1210 LOG.info("Hit Threshold : Terminating");
1211 break;
1212 }
1213 }
1214
1215 LOG.info("Wait For Spool Thread To Finish");
1216 SECONDS.sleep(2);
1217
1218 {
1219 int size = store.getOnDiskSize();
1220 assertTrue(size < (shrinkSize * 1.1));
1221 assertTrue(size > (shrinkSize * 0.9));
1222 }
1223
1224 final int growSize = initialSize * 2;
1225 store.changeDiskCapacity(growSize);
1226 LOG.info("Resized : " + shrinkSize + " ==> " + growSize);
1227
1228 LOG.info("Wait For Spool Thread To Finish");
1229 SECONDS.sleep(2);
1230
1231 for (; ; i++) {
1232 int beforeSize = store.getOnDiskSize();
1233 store.put(new Element(Integer.valueOf(i), new byte[100]));
1234 MILLISECONDS.sleep(500);
1235 int afterSize = store.getOnDiskSize();
1236 LOG.info(beforeSize + " ==> " + afterSize);
1237 if (afterSize <= beforeSize && afterSize > 0.9 * growSize) {
1238 LOG.info("Hit Threshold : Terminating");
1239 break;
1240 }
1241 }
1242
1243 LOG.info("Wait For Spool Thread To Finish");
1244 SECONDS.sleep(2);
1245
1246 {
1247 int size = store.getOnDiskSize();
1248 assertTrue(size < (growSize * 1.1));
1249 assertTrue(size > (growSize * 0.9));
1250 }
1251 }
1252
1253 private DiskStore getDiskStore(Store diskBackedMemoryStore) throws NoSuchFieldException, IllegalAccessException {
1254 Field f = FrontEndCacheTier.class.getDeclaredField("authority");
1255 f.setAccessible(true);
1256 return (DiskStore) f.get(diskBackedMemoryStore);
1257 }
1258
1259 @Test
1260 public void testDiskPersistentExpiryThreadBehavior() {
1261 CacheManager cacheManager = CacheManager.getInstance();
1262 try {
1263 CacheConfiguration configuration = new CacheConfiguration("testCache", 20);
1264 configuration.setOverflowToDisk(true);
1265 configuration.setTimeToIdleSeconds(10);
1266 configuration.setDiskPersistent(true);
1267 configuration.setDiskExpiryThreadIntervalSeconds(1);
1268
1269 Cache cache = new Cache(configuration);
1270 try {
1271 cacheManager.addCache(cache);
1272
1273 cache.put(new Element("1", "A value"));
1274
1275 for (int i = 0; i < 20; i++) {
1276 if (cache.get("1") == null) {
1277 throw new AssertionError();
1278 }
1279
1280 try {
1281 Thread.sleep(1 * 1000);
1282 } catch (InterruptedException e) {
1283
1284 }
1285 }
1286 } finally {
1287 cache.removeAll();
1288 }
1289 } finally {
1290 cacheManager.shutdown();
1291 }
1292 }
1293 }