1 package net.sf.ehcache.store.disk;
2
3 import net.sf.ehcache.Cache;
4 import net.sf.ehcache.CacheException;
5 import net.sf.ehcache.Ehcache;
6 import net.sf.ehcache.Element;
7 import net.sf.ehcache.config.CacheConfiguration;
8 import net.sf.ehcache.event.CacheEventListener;
9 import net.sf.ehcache.pool.Pool;
10 import net.sf.ehcache.pool.impl.ConstantSizeOfEngine;
11 import net.sf.ehcache.pool.impl.FromLargestCacheOnDiskPoolEvictor;
12 import net.sf.ehcache.pool.impl.FromLargestCacheOnHeapPoolEvictor;
13 import net.sf.ehcache.pool.impl.StrictlyBoundedPool;
14 import net.sf.ehcache.store.DefaultElementValueComparator;
15 import org.junit.After;
16 import org.junit.Before;
17 import org.junit.Test;
18
19 import java.util.concurrent.ConcurrentLinkedQueue;
20 import java.util.concurrent.ExecutorService;
21 import java.util.concurrent.Executors;
22 import java.util.concurrent.Future;
23
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotNull;
27 import static org.junit.Assert.assertNull;
28 import static org.junit.Assert.assertTrue;
29
30 /***
31 * @author Ludovic Orban
32 */
33 public class DiskStorePoolingTest {
34
35 private static final int ELEMENT_SIZE_ON_DISK = 308;
36 private static final int ITERATIONS = 100;
37
38 private volatile Cache cache;
39 private volatile Pool onHeapPool;
40 private volatile Pool onDiskPool;
41 private volatile DiskStore diskStore;
42 private volatile Element lastEvicted;
43
44 private void dump() {
45 System.out.println("# # # # # #");
46 System.out.println(diskStore.getSize() + " elements in cache");
47 System.out.println("on heap size: " + onHeapPool.getSize() + ", on disk size: " + onDiskPool.getSize());
48 System.out.println("# # # # # #");
49 }
50
51 @Before
52 public void setUp() {
53 cache = new Cache(new CacheConfiguration("myCache1", 0).eternal(true).diskPersistent(true));
54
55 lastEvicted = null;
56 cache.getCacheEventNotificationService().registerListener(new CacheEventListener() {
57 public void notifyElementRemoved(Ehcache cache, Element element) throws CacheException { }
58
59 public void notifyElementPut(Ehcache cache, Element element) throws CacheException { }
60
61 public void notifyElementUpdated(Ehcache cache, Element element) throws CacheException { }
62
63 public void notifyElementExpired(Ehcache cache, Element element) { }
64
65 public void notifyElementEvicted(Ehcache cache, Element element) {
66 lastEvicted = element;
67 }
68
69 public void notifyRemoveAll(Ehcache cache) { }
70
71 public void dispose() { }
72
73 @Override
74 public Object clone() throws CloneNotSupportedException {
75 return super.clone();
76 }
77 });
78
79 onHeapPool = new StrictlyBoundedPool(
80 16384 * 3,
81 new FromLargestCacheOnHeapPoolEvictor(),
82 new ConstantSizeOfEngine(
83 1536,
84 14336,
85 512
86 )
87 );
88
89 onDiskPool = new StrictlyBoundedPool(
90 ELEMENT_SIZE_ON_DISK * 2,
91 new FromLargestCacheOnDiskPoolEvictor(),
92 null
93 );
94
95 diskStore = DiskStore.create(cache, System.getProperty("java.io.tmpdir"), onHeapPool, onDiskPool);
96 diskStore.removeAll();
97 }
98
99 @After
100 public void tearDown() {
101 cache.dispose();
102 diskStore.dispose();
103 }
104
105 @Test
106 public void testPersistence() throws Exception {
107 for (int i = 0; i < ITERATIONS; i++) {
108 persistence();
109
110 tearDown();
111 setUp();
112 }
113 }
114
115 public void persistence() throws Exception {
116
117 diskStore.put(new Element(1000, "1000"));
118 diskStore.put(new Element(1001, "1001"));
119
120 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
121
122 assertEquals(2, diskStore.getSize());
123 assertEquals(2 * 16384, onHeapPool.getSize());
124 assertEquals(2 * ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
125 assertEquals(2 * 16384, diskStore.getInMemorySizeInBytes());
126 assertEquals(2 * ELEMENT_SIZE_ON_DISK, diskStore.getOnDiskSizeInBytes());
127 diskStore.dispose();
128
129 for (int i = 1000; i < 1030; i++) {
130 diskStore = DiskStore.create(cache, System.getProperty("java.io.tmpdir"), onHeapPool, onDiskPool);
131 assertEquals(2, diskStore.getSize());
132 assertEquals(2 * 16384, onHeapPool.getSize());
133 assertEquals(2 * ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
134 assertEquals(2 * 16384, diskStore.getInMemorySizeInBytes());
135 assertEquals(2 * ELEMENT_SIZE_ON_DISK, diskStore.getOnDiskSizeInBytes());
136 assertEquals(new Element(1000, "1000"), diskStore.get(1000));
137 assertEquals(new Element(1001, "1001"), diskStore.get(1001));
138 diskStore.dispose();
139 }
140 }
141
142 @Test
143 public void testPutNew() throws Exception {
144 for (int i = 0; i < ITERATIONS; i++) {
145 putNew();
146
147 tearDown();
148 setUp();
149 }
150 }
151
152 public void putNew() throws Exception {
153
154 for (int i = 1000; i < 1020; i++) {
155 Element e = new Element(i, "" + i);
156 diskStore.put(e);
157 }
158
159 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
160
161 assertEquals(2, diskStore.getSize());
162 assertEquals(16384 * 2, onHeapPool.getSize());
163 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
164
165
166
167 diskStore.put(new Element(1999, "1999"));
168 assertNotNull(diskStore.get(1999));
169
170 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
171
172 assertEquals(2, diskStore.getSize());
173 assertEquals(16384 * 2, onHeapPool.getSize());
174 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
175 }
176
177 @Test
178 public void testPutManyAtSameKey() throws Exception {
179
180 for (int i = 0; i < 100; i++) {
181 Element e = new Element(1, "" + i);
182 diskStore.put(e);
183 }
184
185 for (int i = 0; i < 1000; i++) {
186 Element element = diskStore.get(1);
187 assertNotNull(element);
188 }
189
190 }
191
192 @Test
193 public void testPutThenRemove() throws Exception {
194 for (int i = 0; i < ITERATIONS; i++) {
195 putThenRemove();
196
197 tearDown();
198 setUp();
199 }
200 }
201
202 public void putThenRemove() throws Exception {
203 for (int i = 1000; i < 2000; i++) {
204 diskStore.put(new Element(i, "" + i));
205 assertTrue(diskStore.getSize() <= 10);
206
207 if (i % 2 == 1) {
208 diskStore.remove(i);
209 }
210 assertTrue(diskStore.getSize() <= 10);
211 }
212
213 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
214
215 assertTrue(diskStore.getSize() >= 1);
216 assertTrue(diskStore.getSize() <= 2);
217 assertTrue(onHeapPool.getSize() >= 16384);
218 assertTrue(onHeapPool.getSize() <= 16384 * 2);
219 assertTrue(onDiskPool.getSize() >= ELEMENT_SIZE_ON_DISK);
220 assertTrue(onDiskPool.getSize() <= ELEMENT_SIZE_ON_DISK * 2);
221
222 for (int i = 1000; i < 2000; i++) {
223 if (i % 2 == 0) {
224 diskStore.remove(i);
225 }
226 }
227
228 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
229
230 assertEquals(0, diskStore.getSize());
231 assertEquals(0, onHeapPool.getSize());
232 assertEquals(0, onDiskPool.getSize());
233 }
234
235
236 @Test
237 public void testPutIfAbsentNew() throws Exception {
238 for (int i = 0; i < ITERATIONS; i++) {
239 putIfAbsentNew();
240
241 tearDown();
242 setUp();
243 }
244 }
245
246 public void putIfAbsentNew() throws Exception {
247
248 for (int i = 1000; i < 1020; i++) {
249 Element e = new Element(i, "" + i);
250 diskStore.putIfAbsent(e);
251 }
252
253 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
254
255 assertEquals(2, diskStore.getSize());
256 assertEquals(16384 * 2, onHeapPool.getSize());
257 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
258
259
260 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
261 diskStore.putIfAbsent(new Element(1999, "1999"));
262 assertNotNull(diskStore.get(1999));
263
264 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
265
266 assertEquals(2, diskStore.getSize());
267 assertEquals(16384 * 2, onHeapPool.getSize());
268 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
269 }
270
271 @Test
272 public void testPutUpdate() throws Exception {
273 for (int i = 0; i < ITERATIONS; i++) {
274 putUpdate();
275
276 tearDown();
277 setUp();
278 }
279 }
280
281 public void putUpdate() throws Exception {
282 diskStore.put(new Element(1001, "1001"));
283 diskStore.put(new Element(1002, "1002"));
284 diskStore.put(new Element(1003, "1003"));
285
286 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
287
288 assertEquals(2, diskStore.getSize());
289 assertEquals(16384 * 2, onHeapPool.getSize());
290 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
291
292
293 Object key = diskStore.getKeys().iterator().next();
294 diskStore.put(new Element(key, key.toString()));
295 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
296
297 assertEquals(2, diskStore.getSize());
298 assertEquals(16384 * 2, onHeapPool.getSize());
299 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
300
301
302
303 key = diskStore.getKeys().iterator().next();
304 diskStore.put(new Element(key, key.toString()));
305
306 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
307
308 assertEquals(2, diskStore.getSize());
309 assertEquals(16384 * 2, onHeapPool.getSize());
310 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
311 }
312
313 @Test
314 public void testPutIfAbsentUpdate() throws Exception {
315 for (int i = 0; i < ITERATIONS; i++) {
316 putIfAbsentUpdate();
317
318 tearDown();
319 setUp();
320 }
321 }
322
323 public void putIfAbsentUpdate() throws Exception {
324
325 assertNull(diskStore.putIfAbsent(new Element(1001, "11#1")));
326 assertNotNull(diskStore.putIfAbsent(new Element(1001, "11#2")));
327 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
328 assertNull(diskStore.putIfAbsent(new Element(1002, "12#1")));
329 assertNotNull(diskStore.putIfAbsent(new Element(1002, "12#2")));
330 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
331
332 assertNull(diskStore.putIfAbsent(new Element(1003, "13#1")));
333 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
334 Element oldElement = diskStore.putIfAbsent(new Element(1003, "13#2"));
335
336 assertNotNull(oldElement);
337 assertEquals(2, diskStore.getSize());
338 assertEquals(16384 * 2, onHeapPool.getSize());
339 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
340 }
341
342 @Test
343 public void testRemove() throws Exception {
344 for (int i = 0; i < ITERATIONS; i++) {
345 remove();
346
347 tearDown();
348 setUp();
349 }
350 }
351
352 public void remove() throws Exception {
353
354 diskStore.put(new Element(1001, "1001"));
355 diskStore.put(new Element(1002, "1002"));
356 diskStore.put(new Element(1003, "1003"));
357
358 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
359
360 assertEquals(2, diskStore.getSize());
361 assertEquals(16384 * 2, onHeapPool.getSize());
362 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
363
364
365 Object key = diskStore.getKeys().iterator().next();
366 diskStore.remove(key);
367
368 assertEquals(1, diskStore.getSize());
369 assertEquals(16384, onHeapPool.getSize());
370 assertEquals(ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
371 }
372
373 @Test
374 public void testReplace1Arg() throws Exception {
375 for (int i = 0; i < ITERATIONS; i++) {
376 replace1Arg();
377
378 tearDown();
379 setUp();
380 }
381 }
382
383 public void replace1Arg() throws Exception {
384
385 diskStore.put(new Element(1001, "11#1"));
386 diskStore.put(new Element(1002, "12#1"));
387 diskStore.put(new Element(1003, "13#1"));
388 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
389
390 assertEquals(2, diskStore.getSize());
391 assertEquals(16384 * 2, onHeapPool.getSize());
392 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
393
394
395 Object key = diskStore.getKeys().iterator().next();
396 Element replaced = diskStore.replace(new Element(key, "22#2"));
397 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
398
399 assertEquals(new Element(key, "22#2"), replaced);
400 assertEquals(2, diskStore.getSize());
401 assertEquals(16384 * 2, onHeapPool.getSize());
402 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
403
404
405 assertNull(diskStore.replace(new Element(1999, 1999 + "19#2")));
406 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
407
408 assertEquals(2, diskStore.getSize());
409 assertEquals(16384 * 2, onHeapPool.getSize());
410 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
411 }
412
413 @Test
414 public void testReplace3Args() throws Exception {
415 for (int i = 0; i < ITERATIONS; i++) {
416 replace3Args();
417
418 tearDown();
419 setUp();
420 }
421 }
422
423 public void replace3Args() throws Exception {
424
425 diskStore.put(new Element(1001, "11#1"));
426 diskStore.put(new Element(1002, "12#1"));
427 diskStore.put(new Element(1003, "13#1"));
428 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
429
430 assertEquals(2, diskStore.getSize());
431 assertEquals(16384 * 2, onHeapPool.getSize());
432 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
433
434
435 Object key = diskStore.getKeys().iterator().next();
436 assertTrue(diskStore.replace(diskStore.getQuiet(key), new Element(key, "20#2"), new DefaultElementValueComparator()));
437 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
438
439 if (lastEvicted.getObjectKey().equals(key)) {
440
441 assertEquals(1, diskStore.getSize());
442 assertEquals(16384, onHeapPool.getSize());
443 assertEquals(ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
444 } else {
445 assertEquals(2, diskStore.getSize());
446 assertEquals(16384 * 2, onHeapPool.getSize());
447 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
448 }
449
450
451 assertFalse(diskStore.replace(new Element(1999, 1999 + "19#1"), new Element(1999, "19#2"), new DefaultElementValueComparator()));
452 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
453
454 assertEquals(2, diskStore.getSize());
455 assertEquals(16384 * 2, onHeapPool.getSize());
456 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
457 }
458
459 @Test
460 public void testRemoveElement() throws Exception {
461 for (int i = 0; i < ITERATIONS; i++) {
462 removeElement();
463
464 tearDown();
465 setUp();
466 }
467 }
468
469 public void removeElement() throws Exception {
470
471 diskStore.put(new Element(1001, "1001"));
472 diskStore.put(new Element(1002, "1002"));
473 diskStore.put(new Element(1003, "1003"));
474 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
475
476 assertEquals(2, diskStore.getSize());
477 assertEquals(16384 * 2, onHeapPool.getSize());
478 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
479
480
481 assertNull(diskStore.removeElement(new Element(1999, 1999 + ""), new DefaultElementValueComparator()));
482
483 assertEquals(2, diskStore.getSize());
484 assertEquals(16384 * 2, onHeapPool.getSize());
485 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
486
487
488 Object key = diskStore.getKeys().iterator().next();
489 assertEquals(new Element(key, key + ""), diskStore.removeElement(new Element(key, key + ""), new DefaultElementValueComparator()));
490
491 assertEquals(1, diskStore.getSize());
492 assertEquals(16384, onHeapPool.getSize());
493 assertEquals(ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
494
495
496 diskStore.put(new Element(1004, "1004"));
497
498 key = diskStore.getKeys().iterator().next();
499 assertEquals(new Element(key, key + ""), diskStore.removeElement(new Element(key, key + ""), new DefaultElementValueComparator()));
500
501 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
502
503 assertEquals(1, diskStore.getSize());
504 assertEquals(16384, onHeapPool.getSize());
505 assertEquals(ELEMENT_SIZE_ON_DISK, onDiskPool.getSize());
506 }
507
508 @Test
509 public void testRemoveAll() throws Exception {
510 for (int i = 0; i < ITERATIONS; i++) {
511 removeAll();
512
513 tearDown();
514 setUp();
515 }
516 }
517
518 public void removeAll() throws Exception {
519
520 diskStore.put(new Element(1001, "1001"));
521 diskStore.put(new Element(1002, "1002"));
522 diskStore.put(new Element(1003, "1003"));
523
524 diskStore.waitUntilEverythingGotFlushedToDisk(3000);
525
526 assertEquals(2, diskStore.getSize());
527 assertEquals(16384 * 2, onHeapPool.getSize());
528 assertEquals(ELEMENT_SIZE_ON_DISK * 2, onDiskPool.getSize());
529
530 diskStore.removeAll();
531
532 assertEquals(0, diskStore.getSize());
533 assertEquals(0, onHeapPool.getSize());
534 assertEquals(0, onDiskPool.getSize());
535 }
536
537 @Test
538 public void testMultithreaded() throws Exception {
539 final int nThreads = 16;
540
541 final ExecutorService executor = Executors.newFixedThreadPool(nThreads);
542 final ConcurrentLinkedQueue<Future<?>> queue = new ConcurrentLinkedQueue<Future<?>>();
543
544 for (int i = 0; i < nThreads; i++) {
545 Future<?> f = executor.submit(new Runnable() {
546 public void run() {
547 for (int i = 0; i < 10000; i++) {
548 Element e = new Element(i, "" + i);
549 diskStore.put(e);
550
551 Thread.yield();
552 if ((i + 1) % 1000 == 0) { diskStore.removeAll(); }
553 }
554 }
555 });
556 queue.add(f);
557 }
558
559 while (!queue.isEmpty()) {
560 Future<?> f = queue.poll();
561 f.get();
562 }
563
564 assertTrue(16384 * 2 >= onHeapPool.getSize());
565 assertTrue(16384 * 2 >= onDiskPool.getSize());
566 assertEquals(onHeapPool.getSize(), onDiskPool.getSize());
567 }
568
569 }