00001 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 00002 * Copyright by The HDF Group. * 00003 * Copyright by the Board of Trustees of the University of Illinois. * 00004 * All rights reserved. * 00005 * * 00006 * This file is part of HDF5. The full HDF5 copyright notice, including * 00007 * terms governing use, modification, and redistribution, is contained in * 00008 * the files COPYING and Copyright.html. COPYING can be found at the root * 00009 * of the source code distribution tree; Copyright.html can be found at the * 00010 * root level of an installed copy of the electronic HDF5 document set and * 00011 * is linked from the top-level documents page. It can also be found at * 00012 * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * 00013 * access to either file, you may request a copy from help@hdfgroup.org. * 00014 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 00015 00016 /*------------------------------------------------------------------------- 00017 * 00018 * Created: H5Cprivate.h 00019 * 6/3/04 00020 * John Mainzer 00021 * 00022 * Purpose: Constants and typedefs available to the rest of the 00023 * library. 00024 * 00025 * Modifications: 00026 * 00027 *------------------------------------------------------------------------- 00028 */ 00029 00030 #ifndef _H5Cprivate_H 00031 #define _H5Cprivate_H 00032 00033 #include "H5Cpublic.h" /*public prototypes */ 00034 00035 /* Pivate headers needed by this header */ 00036 #include "H5private.h" /* Generic Functions */ 00037 #include "H5Fprivate.h" /* File access */ 00038 00039 00040 #define H5C_DO_SANITY_CHECKS 0 00041 #define H5C_DO_EXTREME_SANITY_CHECKS 0 00042 00043 /* This sanity checking constant was picked out of the air. Increase 00044 * or decrease it if appropriate. Its purposes is to detect corrupt 00045 * object sizes, so it probably doesn't matter if it is a bit big. 00046 * 00047 * JRM - 5/17/04 00048 */ 00049 #define H5C_MAX_ENTRY_SIZE ((size_t)(32 * 1024 * 1024)) 00050 00051 /* H5C_COLLECT_CACHE_STATS controls overall collection of statistics 00052 * on cache activity. In general, this #define should be set to 0. 00053 */ 00054 #define H5C_COLLECT_CACHE_STATS 0 00055 00056 /* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics 00057 * in individual cache entries. 00058 * 00059 * H5C_COLLECT_CACHE_ENTRY_STATS should only be defined to true if 00060 * H5C_COLLECT_CACHE_STATS is also defined to true. 00061 */ 00062 #if H5C_COLLECT_CACHE_STATS 00063 00064 #define H5C_COLLECT_CACHE_ENTRY_STATS 1 00065 00066 #else 00067 00068 #define H5C_COLLECT_CACHE_ENTRY_STATS 0 00069 00070 #endif /* H5C_COLLECT_CACHE_STATS */ 00071 00072 00073 #ifdef H5_HAVE_PARALLEL 00074 00075 /* we must maintain the clean and dirty LRU lists when we are compiled 00076 * with parallel support. 00077 */ 00078 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS 1 00079 00080 #else /* H5_HAVE_PARALLEL */ 00081 00082 /* The clean and dirty LRU lists don't buy us anything here -- we may 00083 * want them on for testing on occasion, but in general they should be 00084 * off. 00085 */ 00086 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS 0 00087 00088 #endif /* H5_HAVE_PARALLEL */ 00089 00090 00091 /* Typedef for the main structure for the cache (defined in H5Cpkg.h) */ 00092 00093 typedef struct H5C_t H5C_t; 00094 00095 00096 /* 00097 * Class methods pertaining to caching. Each type of cached object will 00098 * have a constant variable with permanent life-span that describes how 00099 * to cache the object. That variable will be of type H5C_class_t and 00100 * have the following required fields... 00101 * 00102 * LOAD: Loads an object from disk to memory. The function 00103 * should allocate some data structure and return it. 00104 * 00105 * FLUSH: Writes some data structure back to disk. It would be 00106 * wise for the data structure to include dirty flags to 00107 * indicate whether it really needs to be written. This 00108 * function is also responsible for freeing memory allocated 00109 * by the LOAD method if the DEST argument is non-zero (by 00110 * calling the DEST method). 00111 * 00112 * DEST: Just frees memory allocated by the LOAD method. 00113 * 00114 * CLEAR: Just marks object as non-dirty. 00115 * 00116 * SIZE: Report the size (on disk) of the specified cache object. 00117 * Note that the space allocated on disk may not be contiguous. 00118 */ 00119 00120 #define H5C_CALLBACK__NO_FLAGS_SET 0x0 00121 #define H5C_CALLBACK__SIZE_CHANGED_FLAG 0x1 00122 #define H5C_CALLBACK__RENAMED_FLAG 0x2 00123 00124 /* Actions that can be reported to 'notify' client callback */ 00125 typedef enum H5C_notify_action_t { 00126 H5C_NOTIFY_ACTION_AFTER_INSERT, /* Entry has been added to the cache */ 00127 /* (could be loaded from file with 00128 * 'protect' call, or inserted 00129 * with 'set' call) 00130 */ 00131 H5C_NOTIFY_ACTION_BEFORE_EVICT /* Entry is about to be evicted from cache */ 00132 } H5C_notify_action_t; 00133 00134 typedef void *(*H5C_load_func_t)(H5F_t *f, 00135 hid_t dxpl_id, 00136 haddr_t addr, 00137 const void *udata1, 00138 void *udata2); 00139 typedef herr_t (*H5C_flush_func_t)(H5F_t *f, 00140 hid_t dxpl_id, 00141 hbool_t dest, 00142 haddr_t addr, 00143 void *thing, 00144 unsigned * flags_ptr); 00145 typedef herr_t (*H5C_dest_func_t)(H5F_t *f, 00146 void *thing); 00147 typedef herr_t (*H5C_clear_func_t)(H5F_t *f, 00148 void *thing, 00149 hbool_t dest); 00150 typedef herr_t (*H5C_notify_func_t)(H5C_notify_action_t action, 00151 void *thing); 00152 typedef herr_t (*H5C_size_func_t)(const H5F_t *f, 00153 const void *thing, 00154 size_t *size_ptr); 00155 00156 typedef struct H5C_class_t { 00157 int id; 00158 H5C_load_func_t load; 00159 H5C_flush_func_t flush; 00160 H5C_dest_func_t dest; 00161 H5C_clear_func_t clear; 00162 H5C_size_func_t size; 00163 } H5C_class_t; 00164 00165 00166 /* Type defintions of call back functions used by the cache as a whole */ 00167 00168 typedef herr_t (*H5C_write_permitted_func_t)(const H5F_t *f, 00169 hid_t dxpl_id, 00170 hbool_t * write_permitted_ptr); 00171 00172 typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr, 00173 haddr_t addr, 00174 hbool_t was_dirty, 00175 unsigned flags, 00176 int type_id); 00177 00178 /* Upper and lower limits on cache size. These limits are picked 00179 * out of a hat -- you should be able to change them as necessary. 00180 * 00181 * However, if you need a very big cache, you should also increase the 00182 * size of the hash table (H5C__HASH_TABLE_LEN in H5Cpkg.h). The current 00183 * upper bound on cache size is rather large for the current hash table 00184 * size. 00185 */ 00186 00187 #define H5C__MAX_MAX_CACHE_SIZE ((size_t)(128 * 1024 * 1024)) 00188 #define H5C__MIN_MAX_CACHE_SIZE ((size_t)(1024)) 00189 00190 00191 /* Default max cache size and min clean size are give here to make 00192 * them generally accessable. 00193 */ 00194 00195 #define H5C__DEFAULT_MAX_CACHE_SIZE ((size_t)(4 * 1024 * 1024)) 00196 #define H5C__DEFAULT_MIN_CLEAN_SIZE ((size_t)(2 * 1024 * 1024)) 00197 00198 00199 /**************************************************************************** 00200 * 00201 * structure H5C_cache_entry_t 00202 * 00203 * Instances of the H5C_cache_entry_t structure are used to store cache 00204 * entries in a hash table and sometimes in a skip list. 00205 * See H5SL.c for the particulars of the skip list. 00206 * 00207 * In typical application, this structure is the first field in a 00208 * structure to be cached. For historical reasons, the external module 00209 * is responsible for managing the is_dirty field (this is no longer 00210 * completely true. See the comment on the is_dirty field for details). 00211 * All other fields are managed by the cache. 00212 * 00213 * The fields of this structure are discussed individually below: 00214 * 00215 * JRM - 4/26/04 00216 * 00217 * magic: Unsigned 32 bit integer that must always be set to 00218 * H5C__H5C_CACHE_ENTRY_T_MAGIC when the entry is valid. 00219 * The field must be set to H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 00220 * just before the entry is freed. 00221 * 00222 * This is necessary, as the LRU list can be changed out 00223 * from under H5C_make_space_in_cache() by the flush 00224 * callback which may change the size of an existing entry, 00225 * and/or load a new entry while serializing the target entry. 00226 * 00227 * This in turn can cause a recursive call to 00228 * H5C_make_space_in_cache() which may either flush or evict 00229 * the next entry that the first invocation of that function 00230 * was about to examine. 00231 * 00232 * The magic field allows H5C_make_space_in_cache() to 00233 * detect this case, and re-start its scan from the bottom 00234 * of the LRU when this situation occurs. 00235 * 00236 * This field is only compiled in debug mode. 00237 * 00238 * addr: Base address of the cache entry on disk. 00239 * 00240 * size: Length of the cache entry on disk. Note that unlike normal 00241 * caches, the entries in this cache are of variable length. 00242 * The entries should never overlap, and when we do writebacks, 00243 * we will want to writeback adjacent entries where possible. 00244 * 00245 * NB: At present, entries need not be contiguous on disk. Until 00246 * we fix this, we can't do much with writing back adjacent 00247 * entries. 00248 * 00249 * type: Pointer to the instance of H5C_class_t containing pointers 00250 * to the methods for cache entries of the current type. This 00251 * field should be NULL when the instance of H5C_cache_entry_t 00252 * is not in use. 00253 * 00254 * The name is not particularly descriptive, but is retained 00255 * to avoid changes in existing code. 00256 * 00257 * is_dirty: Boolean flag indicating whether the contents of the cache 00258 * entry has been modified since the last time it was written 00259 * to disk. 00260 * 00261 * NOTE: For historical reasons, this field is not maintained 00262 * by the cache. Instead, the module using the cache 00263 * sets this flag when it modifies the entry, and the 00264 * flush and clear functions supplied by that module 00265 * reset the dirty when appropriate. 00266 * 00267 * This is a bit quirky, so we may want to change this 00268 * someday. However it will require a change in the 00269 * cache interface. 00270 * 00271 * Update: Management of the is_dirty field has been largely 00272 * moved into the cache. The only remaining exceptions 00273 * are the flush and clear functions supplied by the 00274 * modules using the cache. These still clear the 00275 * is_dirty field as before. -- JRM 7/5/05 00276 * 00277 * dirtied: Boolean flag used to indicate that the entry has been 00278 * dirtied while protected. 00279 * 00280 * This field is set to FALSE in the protect call, and may 00281 * be set to TRUE by the 00282 * H5C_mark_pinned_or_protected_entry_dirty() 00283 * call at an time prior to the unprotect call. 00284 * 00285 * The H5C_mark_pinned_or_protected_entry_dirty() call exists 00286 * as a convenience function for the fractal heap code which 00287 * may not know if an entry is protected or pinned, but knows 00288 * that is either protected or pinned. The dirtied field was 00289 * added as in the parallel case, it is necessary to know 00290 * whether a protected entry was dirty prior to the protect call. 00291 * 00292 * is_protected: Boolean flag indicating whether this entry is protected 00293 * (or locked, to use more conventional terms). When it is 00294 * protected, the entry cannot be flushed or accessed until 00295 * it is unprotected (or unlocked -- again to use more 00296 * conventional terms). 00297 * 00298 * Note that protected entries are removed from the LRU lists 00299 * and inserted on the protected list. 00300 * 00301 * is_read_only: Boolean flag that is only meaningful if is_protected is 00302 * TRUE. In this circumstance, it indicates whether the 00303 * entry has been protected read only, or read/write. 00304 * 00305 * If the entry has been protected read only (i.e. is_protected 00306 * and is_read_only are both TRUE), we allow the entry to be 00307 * protected more than once. 00308 * 00309 * In this case, the number of readers is maintained in the 00310 * ro_ref_count field (see below), and unprotect calls simply 00311 * decrement that field until it drops to zero, at which point 00312 * the entry is actually unprotected. 00313 * 00314 * ro_ref_count: Integer field used to maintain a count of the number of 00315 * outstanding read only protects on this entry. This field 00316 * must be zero whenever either is_protected or is_read_only 00317 * are TRUE. 00318 * 00319 * is_pinned: Boolean flag indicating whether the entry has been pinned 00320 * in the cache. 00321 * 00322 * For very hot entries, the protect / unprotect overhead 00323 * can become excessive. Thus the cache has been extended 00324 * to allow an entry to be "pinned" in the cache. 00325 * 00326 * Pinning an entry in the cache has several implications: 00327 * 00328 * 1) A pinned entry cannot be evicted. Thus unprotected 00329 * pinned entries must be stored in the pinned entry 00330 * list, instead of being managed by the replacement 00331 * policy code (LRU at present). 00332 * 00333 * 2) A pinned entry can be accessed or modified at any time. 00334 * Therefore, the cache must check with the entry owner 00335 * before flushing it. If permission is denied, the 00336 * cache does not flush the entry. 00337 * 00338 * 3) A pinned entry can be marked as dirty (and possibly 00339 * change size) while it is unprotected. 00340 * 00341 * 4) The flush-destroy code must allow pinned entries to 00342 * be unpinned (and possibly unprotected) during the 00343 * flush. 00344 * 00345 * JRM -- 3/16/06 00346 * 00347 * in_slist: Boolean flag indicating whether the entry is in the skip list 00348 * As a general rule, entries are placed in the list when they 00349 * are marked dirty. However they may remain in the list after 00350 * being flushed. 00351 * 00352 * Update: Dirty entries are now removed from the skip list 00353 * when they are flushed. 00354 * 00355 * flush_marker: Boolean flag indicating that the entry is to be flushed 00356 * the next time H5C_flush_cache() is called with the 00357 * H5AC__FLUSH_MARKED_ENTRIES_FLAG. The flag is reset when 00358 * the entry is flushed for whatever reason. 00359 * 00360 * clear_on_unprotect: Boolean flag used only in PHDF5. When H5C is used 00361 * to implement the metadata cache In the parallel case, only 00362 * the cache with mpi rank 0 is allowed to actually write to 00363 * file -- all other caches must retain dirty entries until they 00364 * are advised that the entry is clean. 00365 * 00366 * This flag is used in the case that such an advisory is 00367 * received when the entry is protected. If it is set when an 00368 * entry is unprotected, and the dirtied flag is not set in 00369 * the unprotect, the entry's is_dirty flag is reset by flushing 00370 * it with the H5C__FLUSH_CLEAR_ONLY_FLAG. 00371 * 00372 * flush_in_progress: Boolean flag that is set to true iff the entry 00373 * is in the process of being flushed. This allows the cache 00374 * to detect when a call is the result of a flush callback. 00375 * 00376 * destroy_in_progress: Boolean flag that is set to true iff the entry 00377 * is in the process of being flushed and destroyed. 00378 * 00379 * free_file_space_on_destroy: Boolean flag that is set to true iff the entry 00380 * is in the process of being flushed and destroyed and the file 00381 * space used by the object should be freed by the cache client's 00382 * 'dest' callback routine. 00383 * 00384 * 00385 * Fields supporting the hash table: 00386 * 00387 * Fields in the cache are indexed by a more or less conventional hash table. 00388 * If there are multiple entries in any hash bin, they are stored in a doubly 00389 * linked list. 00390 * 00391 * ht_next: Next pointer used by the hash table to store multiple 00392 * entries in a single hash bin. This field points to the 00393 * next entry in the doubly linked list of entries in the 00394 * hash bin, or NULL if there is no next entry. 00395 * 00396 * ht_prev: Prev pointer used by the hash table to store multiple 00397 * entries in a single hash bin. This field points to the 00398 * previous entry in the doubly linked list of entries in 00399 * the hash bin, or NULL if there is no previuos entry. 00400 * 00401 * 00402 * Fields supporting replacement policies: 00403 * 00404 * The cache must have a replacement policy, and it will usually be 00405 * necessary for this structure to contain fields supporting that policy. 00406 * 00407 * While there has been interest in several replacement policies for 00408 * this cache, the initial development schedule is tight. Thus I have 00409 * elected to support only a modified LRU policy for the first cut. 00410 * 00411 * When additional replacement policies are added, the fields in this 00412 * section will be used in different ways or not at all. Thus the 00413 * documentation of these fields is repeated for each replacement policy. 00414 * 00415 * Modified LRU: 00416 * 00417 * When operating in parallel mode, we must ensure that a read does not 00418 * cause a write. If it does, the process will hang, as the write will 00419 * be collective and the other processes will not know to participate. 00420 * 00421 * To deal with this issue, I have modified the usual LRU policy by adding 00422 * clean and dirty LRU lists to the usual LRU list. When reading in 00423 * parallel mode, we evict from the clean LRU list only. This implies 00424 * that we must try to ensure that the clean LRU list is reasonably well 00425 * stocked. See the comments on H5C_t in H5Cpkg.h for more details. 00426 * 00427 * Note that even if we start with a completely clean cache, a sequence 00428 * of protects without unprotects can empty the clean LRU list. In this 00429 * case, the cache must grow temporarily. At the next write, we will 00430 * attempt to evict enough entries to get the cache down to its nominal 00431 * maximum size. 00432 * 00433 * The use of the replacement policy fields under the Modified LRU policy 00434 * is discussed below: 00435 * 00436 * next: Next pointer in either the LRU or the protected list, 00437 * depending on the current value of protected. If there 00438 * is no next entry on the list, this field should be set 00439 * to NULL. 00440 * 00441 * prev: Prev pointer in either the LRU or the protected list, 00442 * depending on the current value of protected. If there 00443 * is no previous entry on the list, this field should be 00444 * set to NULL. 00445 * 00446 * aux_next: Next pointer on either the clean or dirty LRU lists. 00447 * This entry should be NULL when protected is true. When 00448 * protected is false, and dirty is true, it should point 00449 * to the next item on the dirty LRU list. When protected 00450 * is false, and dirty is false, it should point to the 00451 * next item on the clean LRU list. In either case, when 00452 * there is no next item, it should be NULL. 00453 * 00454 * aux_prev: Previous pointer on either the clean or dirty LRU lists. 00455 * This entry should be NULL when protected is true. When 00456 * protected is false, and dirty is true, it should point 00457 * to the previous item on the dirty LRU list. When protected 00458 * is false, and dirty is false, it should point to the 00459 * previous item on the clean LRU list. In either case, when 00460 * there is no previous item, it should be NULL. 00461 * 00462 * 00463 * Cache entry stats collection fields: 00464 * 00465 * These fields should only be compiled in when both H5C_COLLECT_CACHE_STATS 00466 * and H5C_COLLECT_CACHE_ENTRY_STATS are true. When present, they allow 00467 * collection of statistics on individual cache entries. 00468 * 00469 * accesses: int32_t containing the number of times this cache entry has 00470 * been referenced in its lifetime. 00471 * 00472 * clears: int32_t containing the number of times this cache entry has 00473 * been cleared in its life time. 00474 * 00475 * flushes: int32_t containing the number of times this cache entry has 00476 * been flushed to file in its life time. 00477 * 00478 * pins: int32_t containing the number of times this cache entry has 00479 * been pinned in cache in its life time. 00480 * 00481 ****************************************************************************/ 00482 00483 #ifndef NDEBUG 00484 #define H5C__H5C_CACHE_ENTRY_T_MAGIC 0x005CAC0A 00485 #define H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef 00486 #endif /* NDEBUG */ 00487 00488 typedef struct H5C_cache_entry_t 00489 { 00490 #ifndef NDEBUG 00491 uint32_t magic; 00492 #endif /* NDEBUG */ 00493 H5C_t * cache_ptr; 00494 haddr_t addr; 00495 size_t size; 00496 const H5C_class_t * type; 00497 hbool_t is_dirty; 00498 hbool_t dirtied; 00499 hbool_t is_protected; 00500 hbool_t is_read_only; 00501 int ro_ref_count; 00502 hbool_t is_pinned; 00503 hbool_t in_slist; 00504 hbool_t flush_marker; 00505 #ifdef H5_HAVE_PARALLEL 00506 hbool_t clear_on_unprotect; 00507 #endif /* H5_HAVE_PARALLEL */ 00508 hbool_t flush_in_progress; 00509 hbool_t destroy_in_progress; 00510 hbool_t free_file_space_on_destroy; 00511 00512 /* fields supporting the hash table: */ 00513 00514 struct H5C_cache_entry_t * ht_next; 00515 struct H5C_cache_entry_t * ht_prev; 00516 00517 /* fields supporting replacement policies: */ 00518 00519 struct H5C_cache_entry_t * next; 00520 struct H5C_cache_entry_t * prev; 00521 struct H5C_cache_entry_t * aux_next; 00522 struct H5C_cache_entry_t * aux_prev; 00523 00524 #if H5C_COLLECT_CACHE_ENTRY_STATS 00525 00526 /* cache entry stats fields */ 00527 00528 int32_t accesses; 00529 int32_t clears; 00530 int32_t flushes; 00531 int32_t pins; 00532 00533 #endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ 00534 00535 } H5C_cache_entry_t; 00536 00537 00538 /**************************************************************************** 00539 * 00540 * structure H5C_auto_size_ctl_t 00541 * 00542 * Instances of H5C_auto_size_ctl_t are used to get and set the control 00543 * fields for automatic cache re-sizing. 00544 * 00545 * The fields of the structure are discussed individually below: 00546 * 00547 * version: Integer field containing the version number of this version 00548 * of the H5C_auto_size_ctl_t structure. Any instance of 00549 * H5C_auto_size_ctl_t passed to the cache must have a known 00550 * version number, or an error will be flagged. 00551 * 00552 * report_fcn: Pointer to the function that is to be called to report 00553 * activities each time the auto cache resize code is executed. If the 00554 * field is NULL, no call is made. 00555 * 00556 * If the field is not NULL, it must contain the address of a function 00557 * of type H5C_auto_resize_report_fcn. 00558 * 00559 * set_initial_size: Boolean flag indicating whether the size of the 00560 * initial size of the cache is to be set to the value given in 00561 * the initial_size field. If set_initial_size is FALSE, the 00562 * initial_size field is ignored. 00563 * 00564 * initial_size: If enabled, this field contain the size the cache is 00565 * to be set to upon receipt of this structure. Needless to say, 00566 * initial_size must lie in the closed interval [min_size, max_size]. 00567 * 00568 * min_clean_fraction: double in the range 0 to 1 indicating the fraction 00569 * of the cache that is to be kept clean. This field is only used 00570 * in parallel mode. Typical values are 0.1 to 0.5. 00571 * 00572 * max_size: Maximum size to which the cache can be adjusted. The 00573 * supplied value must fall in the closed interval 00574 * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, max_size must 00575 * be greater than or equal to min_size. 00576 * 00577 * min_size: Minimum size to which the cache can be adjusted. The 00578 * supplied value must fall in the closed interval 00579 * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, min_size must 00580 * be less than or equal to max_size. 00581 * 00582 * epoch_length: Number of accesses on the cache over which to collect 00583 * hit rate stats before running the automatic cache resize code, 00584 * if it is enabled. 00585 * 00586 * At the end of an epoch, we discard prior hit rate data and start 00587 * collecting afresh. The epoch_length must lie in the closed 00588 * interval [H5C__MIN_AR_EPOCH_LENGTH, H5C__MAX_AR_EPOCH_LENGTH]. 00589 * 00590 * 00591 * Cache size increase control fields: 00592 * 00593 * incr_mode: Instance of the H5C_cache_incr_mode enumerated type whose 00594 * value indicates how we determine whether the cache size should be 00595 * increased. At present there are two possible values: 00596 * 00597 * H5C_incr__off: Don't attempt to increase the size of the cache 00598 * automatically. 00599 * 00600 * When this increment mode is selected, the remaining fields 00601 * in the cache size increase section ar ignored. 00602 * 00603 * H5C_incr__threshold: Attempt to increase the size of the cache 00604 * whenever the average hit rate over the last epoch drops 00605 * below the value supplied in the lower_hr_threshold 00606 * field. 00607 * 00608 * Note that this attempt will fail if the cache is already 00609 * at its maximum size, or if the cache is not already using 00610 * all available space. 00611 * 00612 * lower_hr_threshold: Lower hit rate threshold. If the increment mode 00613 * (incr_mode) is H5C_incr__threshold and the hit rate drops below the 00614 * value supplied in this field in an epoch, increment the cache size by 00615 * size_increment. Note that cache size may not be incremented above 00616 * max_size, and that the increment may be further restricted by the 00617 * max_increment field if it is enabled. 00618 * 00619 * When enabled, this field must contain a value in the range [0.0, 1.0]. 00620 * Depending on the incr_mode selected, it may also have to be less than 00621 * upper_hr_threshold. 00622 * 00623 * increment: Double containing the multiplier used to derive the new 00624 * cache size from the old if a cache size increment is triggered. 00625 * The increment must be greater than 1.0, and should not exceed 2.0. 00626 * 00627 * The new cache size is obtained by multiplying the current max cache 00628 * size by the increment, and then clamping to max_size and to stay 00629 * within the max_increment as necessary. 00630 * 00631 * apply_max_increment: Boolean flag indicating whether the max_increment 00632 * field should be used to limit the maximum cache size increment. 00633 * 00634 * max_increment: If enabled by the apply_max_increment field described 00635 * above, this field contains the maximum number of bytes by which the 00636 * cache size can be increased in a single re-size. 00637 * 00638 * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated 00639 * type whose value indicates whether and by what algorithm we should 00640 * make flash increases in the size of the cache to accomodate insertion 00641 * of large entries and large increases in the size of a single entry. 00642 * 00643 * The addition of the flash increment mode was occasioned by performance 00644 * problems that appear when a local heap is increased to a size in excess 00645 * of the current cache size. While the existing re-size code dealt with 00646 * this eventually, performance was very bad for the remainder of the 00647 * epoch. 00648 * 00649 * At present, there are two possible values for the flash_incr_mode: 00650 * 00651 * H5C_flash_incr__off: Don't perform flash increases in the size of 00652 * the cache. 00653 * 00654 * H5C_flash_incr__add_space: Let x be either the size of a newly 00655 * newly inserted entry, or the number of bytes by which the 00656 * size of an existing entry has been increased. 00657 * 00658 * If 00659 * x > flash_threshold * current max cache size, 00660 * 00661 * increase the current maximum cache size by x * flash_multiple 00662 * less any free space in the cache, and start a new epoch. For 00663 * now at least, pay no attention to the maximum increment. 00664 * 00665 * 00666 * With a little thought, it should be obvious that the above flash 00667 * cache size increase algorithm is not sufficient for all circumstances -- 00668 * for example, suppose the user round robins through 00669 * (1/flash_threshold) +1 groups, adding one data set to each on each 00670 * pass. Then all will increase in size at about the same time, requiring 00671 * the max cache size to at least double to maintain acceptable 00672 * performance, however the above flash increment algorithm will not be 00673 * triggered. 00674 * 00675 * Hopefully, the add space algorithm detailed above will be sufficient 00676 * for the performance problems encountered to date. However, we should 00677 * expect to revisit the issue. 00678 * 00679 * flash_multiple: Double containing the multiple described above in the 00680 * H5C_flash_incr__add_space section of the discussion of the 00681 * flash_incr_mode section. This field is ignored unless flash_incr_mode 00682 * is H5C_flash_incr__add_space. 00683 * 00684 * flash_threshold: Double containing the factor by which current max cache size 00685 * is multiplied to obtain the size threshold for the add_space flash 00686 * increment algorithm. The field is ignored unless flash_incr_mode is 00687 * H5C_flash_incr__add_space. 00688 * 00689 * 00690 * Cache size decrease control fields: 00691 * 00692 * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose 00693 * value indicates how we determine whether the cache size should be 00694 * decreased. At present there are four possibilities. 00695 * 00696 * H5C_decr__off: Don't attempt to decrease the size of the cache 00697 * automatically. 00698 * 00699 * When this increment mode is selected, the remaining fields 00700 * in the cache size decrease section are ignored. 00701 * 00702 * H5C_decr__threshold: Attempt to decrease the size of the cache 00703 * whenever the average hit rate over the last epoch rises 00704 * above the value supplied in the upper_hr_threshold 00705 * field. 00706 * 00707 * H5C_decr__age_out: At the end of each epoch, search the cache for 00708 * entries that have not been accessed for at least the number 00709 * of epochs specified in the epochs_before_eviction field, and 00710 * evict these entries. Conceptually, the maximum cache size 00711 * is then decreased to match the new actual cache size. However, 00712 * this reduction may be modified by the min_size, the 00713 * max_decrement, and/or the empty_reserve. 00714 * 00715 * H5C_decr__age_out_with_threshold: Same as age_out, but we only 00716 * attempt to reduce the cache size when the hit rate observed 00717 * over the last epoch exceeds the value provided in the 00718 * upper_hr_threshold field. 00719 * 00720 * upper_hr_threshold: Upper hit rate threshold. The use of this field 00721 * varies according to the current decr_mode: 00722 * 00723 * H5C_decr__off or H5C_decr__age_out: The value of this field is 00724 * ignored. 00725 * 00726 * H5C_decr__threshold: If the hit rate exceeds this threshold in any 00727 * epoch, attempt to decrement the cache size by size_decrement. 00728 * 00729 * Note that cache size may not be decremented below min_size. 00730 * 00731 * Note also that if the upper_threshold is 1.0, the cache size 00732 * will never be reduced. 00733 * 00734 * H5C_decr__age_out_with_threshold: If the hit rate exceeds this 00735 * threshold in any epoch, attempt to reduce the cache size 00736 * by evicting entries that have not been accessed for more 00737 * than the specified number of epochs. 00738 * 00739 * decrement: This field is only used when the decr_mode is 00740 * H5C_decr__threshold. 00741 * 00742 * The field is a double containing the multiplier used to derive the 00743 * new cache size from the old if a cache size decrement is triggered. 00744 * The decrement must be in the range 0.0 (in which case the cache will 00745 * try to contract to its minimum size) to 1.0 (in which case the 00746 * cache will never shrink). 00747 * 00748 * apply_max_decrement: Boolean flag used to determine whether decrements 00749 * in cache size are to be limited by the max_decrement field. 00750 * 00751 * max_decrement: Maximum number of bytes by which the cache size can be 00752 * decreased in a single re-size. Note that decrements may also be 00753 * restricted by the min_size of the cache, and (in age out modes) by 00754 * the empty_reserve field. 00755 * 00756 * epochs_before_eviction: Integer field used in H5C_decr__age_out and 00757 * H5C_decr__age_out_with_threshold decrement modes. 00758 * 00759 * This field contains the number of epochs an entry must remain 00760 * unaccessed before it is evicted in an attempt to reduce the 00761 * cache size. If applicable, this field must lie in the range 00762 * [1, H5C__MAX_EPOCH_MARKERS]. 00763 * 00764 * apply_empty_reserve: Boolean field controlling whether the empty_reserve 00765 * field is to be used in computing the new cache size when the 00766 * decr_mode is H5C_decr__age_out or H5C_decr__age_out_with_threshold. 00767 * 00768 * empty_reserve: To avoid a constant racheting down of cache size by small 00769 * amounts in the H5C_decr__age_out and H5C_decr__age_out_with_threshold 00770 * modes, this field allows one to require that any cache size 00771 * reductions leave the specified fraction of unused space in the cache. 00772 * 00773 * The value of this field must be in the range [0.0, 1.0]. I would 00774 * expect typical values to be in the range of 0.01 to 0.1. 00775 * 00776 ****************************************************************************/ 00777 00778 #define H5C_RESIZE_CFG__VALIDATE_GENERAL 0x1 00779 #define H5C_RESIZE_CFG__VALIDATE_INCREMENT 0x2 00780 #define H5C_RESIZE_CFG__VALIDATE_DECREMENT 0x4 00781 #define H5C_RESIZE_CFG__VALIDATE_INTERACTIONS 0x8 00782 #define H5C_RESIZE_CFG__VALIDATE_ALL \ 00783 ( \ 00784 H5C_RESIZE_CFG__VALIDATE_GENERAL | \ 00785 H5C_RESIZE_CFG__VALIDATE_INCREMENT | \ 00786 H5C_RESIZE_CFG__VALIDATE_DECREMENT | \ 00787 H5C_RESIZE_CFG__VALIDATE_INTERACTIONS \ 00788 ) 00789 00790 #define H5C__CURR_AUTO_SIZE_CTL_VER 1 00791 #define H5C__CURR_AUTO_RESIZE_RPT_FCN_VER 1 00792 00793 #define H5C__MAX_EPOCH_MARKERS 10 00794 00795 #define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999 00796 #define H5C__DEF_AR_LOWER_THRESHHOLD 0.9 00797 #define H5C__DEF_AR_MAX_SIZE ((size_t)(16 * 1024 * 1024)) 00798 #define H5C__DEF_AR_INIT_SIZE ((size_t)( 1 * 1024 * 1024)) 00799 #define H5C__DEF_AR_MIN_SIZE ((size_t)( 1 * 1024 * 1024)) 00800 #define H5C__DEF_AR_MIN_CLEAN_FRAC 0.5 00801 #define H5C__DEF_AR_INCREMENT 2.0 00802 #define H5C__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024)) 00803 #define H5C__DEF_AR_FLASH_MULTIPLE 1.0 00804 #define H5C__DEV_AR_FLASH_THRESHOLD 0.25 00805 #define H5C__DEF_AR_DECREMENT 0.9 00806 #define H5C__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024)) 00807 #define H5C__DEF_AR_EPCHS_B4_EVICT 3 00808 #define H5C__DEF_AR_EMPTY_RESERVE 0.05 00809 #define H5C__MIN_AR_EPOCH_LENGTH 100 00810 #define H5C__DEF_AR_EPOCH_LENGTH 50000 00811 #define H5C__MAX_AR_EPOCH_LENGTH 1000000 00812 00813 enum H5C_resize_status 00814 { 00815 in_spec, 00816 increase, 00817 flash_increase, 00818 decrease, 00819 at_max_size, 00820 at_min_size, 00821 increase_disabled, 00822 decrease_disabled, 00823 not_full 00824 }; /* enum H5C_resize_conditions */ 00825 00826 typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr, 00827 int32_t version, 00828 double hit_rate, 00829 enum H5C_resize_status status, 00830 size_t old_max_cache_size, 00831 size_t new_max_cache_size, 00832 size_t old_min_clean_size, 00833 size_t new_min_clean_size); 00834 00835 typedef struct H5C_auto_size_ctl_t 00836 { 00837 /* general configuration fields: */ 00838 int32_t version; 00839 H5C_auto_resize_rpt_fcn rpt_fcn; 00840 00841 hbool_t set_initial_size; 00842 size_t initial_size; 00843 00844 double min_clean_fraction; 00845 00846 size_t max_size; 00847 size_t min_size; 00848 00849 int64_t epoch_length; 00850 00851 00852 /* size increase control fields: */ 00853 enum H5C_cache_incr_mode incr_mode; 00854 00855 double lower_hr_threshold; 00856 00857 double increment; 00858 00859 hbool_t apply_max_increment; 00860 size_t max_increment; 00861 00862 enum H5C_cache_flash_incr_mode flash_incr_mode; 00863 double flash_multiple; 00864 double flash_threshold; 00865 00866 /* size decrease control fields: */ 00867 enum H5C_cache_decr_mode decr_mode; 00868 00869 double upper_hr_threshold; 00870 00871 double decrement; 00872 00873 hbool_t apply_max_decrement; 00874 size_t max_decrement; 00875 00876 int32_t epochs_before_eviction; 00877 00878 hbool_t apply_empty_reserve; 00879 double empty_reserve; 00880 00881 } H5C_auto_size_ctl_t; 00882 00883 00884 /* 00885 * Library prototypes. 00886 */ 00887 00888 /* #defines of flags used in the flags parameters in some of the 00889 * following function calls. Note that not all flags are applicable 00890 * to all function calls. Flags that don't apply to a particular 00891 * function are ignored in that function. 00892 * 00893 * These flags apply to all function calls: 00894 * 00895 * H5C__NO_FLAGS_SET (generic "no flags set" for all fcn calls) 00896 * 00897 * 00898 * These flags apply to H5C_insert_entry(): 00899 * 00900 * H5C__SET_FLUSH_MARKER_FLAG 00901 * H5C__PIN_ENTRY_FLAG 00902 * 00903 * These flags apply to H5C_protect() 00904 * 00905 * H5C__READ_ONLY_FLAG 00906 * 00907 * These flags apply to H5C_unprotect(): 00908 * 00909 * H5C__SET_FLUSH_MARKER_FLAG 00910 * H5C__DELETED_FLAG 00911 * H5C__DIRTIED_FLAG 00912 * H5C__SIZE_CHANGED_FLAG 00913 * H5C__PIN_ENTRY_FLAG 00914 * H5C__UNPIN_ENTRY_FLAG 00915 * H5C__FREE_FILE_SPACE_FLAG 00916 * H5C__TAKE_OWNERSHIP_FLAG 00917 * 00918 * These flags apply to H5C_expunge_entry(): 00919 * 00920 * H5C__FREE_FILE_SPACE_FLAG 00921 * 00922 * These flags apply to H5C_flush_cache(): 00923 * 00924 * H5C__FLUSH_INVALIDATE_FLAG 00925 * H5C__FLUSH_CLEAR_ONLY_FLAG 00926 * H5C__FLUSH_MARKED_ENTRIES_FLAG 00927 * H5C__FLUSH_IGNORE_PROTECTED_FLAG (can't use this flag in combination 00928 * with H5C__FLUSH_INVALIDATE_FLAG) 00929 * 00930 * These flags apply to H5C_flush_single_entry(): 00931 * 00932 * H5C__FLUSH_INVALIDATE_FLAG 00933 * H5C__FLUSH_CLEAR_ONLY_FLAG 00934 * H5C__FLUSH_MARKED_ENTRIES_FLAG 00935 * H5C__TAKE_OWNERSHIP_FLAG 00936 */ 00937 00938 #define H5C__NO_FLAGS_SET 0x0000 00939 #define H5C__SET_FLUSH_MARKER_FLAG 0x0001 00940 #define H5C__DELETED_FLAG 0x0002 00941 #define H5C__DIRTIED_FLAG 0x0004 00942 #define H5C__SIZE_CHANGED_FLAG 0x0008 00943 #define H5C__PIN_ENTRY_FLAG 0x0010 00944 #define H5C__UNPIN_ENTRY_FLAG 0x0020 00945 #define H5C__FLUSH_INVALIDATE_FLAG 0x0040 00946 #define H5C__FLUSH_CLEAR_ONLY_FLAG 0x0080 00947 #define H5C__FLUSH_MARKED_ENTRIES_FLAG 0x0100 00948 #define H5C__FLUSH_IGNORE_PROTECTED_FLAG 0x0200 00949 #define H5C__READ_ONLY_FLAG 0x0400 00950 #define H5C__FREE_FILE_SPACE_FLAG 0x0800 00951 #define H5C__TAKE_OWNERSHIP_FLAG 0x1000 00952 00953 00954 H5_DLL H5C_t * H5C_create(size_t max_cache_size, 00955 size_t min_clean_size, 00956 int max_type_id, 00957 const char * (* type_name_table_ptr), 00958 H5C_write_permitted_func_t check_write_permitted, 00959 hbool_t write_permitted, 00960 H5C_log_flush_func_t log_flush, 00961 void * aux_ptr); 00962 00963 H5_DLL void H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr, 00964 int32_t version, 00965 double hit_rate, 00966 enum H5C_resize_status status, 00967 size_t old_max_cache_size, 00968 size_t new_max_cache_size, 00969 size_t old_min_clean_size, 00970 size_t new_min_clean_size); 00971 00972 H5_DLL herr_t H5C_dest(H5F_t * f, 00973 hid_t primary_dxpl_id, 00974 hid_t secondary_dxpl_id); 00975 00976 H5_DLL herr_t H5C_expunge_entry(H5F_t * f, 00977 hid_t primary_dxpl_id, 00978 hid_t secondary_dxpl_id, 00979 const H5C_class_t * type, 00980 haddr_t addr, 00981 unsigned flags); 00982 00983 H5_DLL herr_t H5C_flush_cache(H5F_t * f, 00984 hid_t primary_dxpl_id, 00985 hid_t secondary_dxpl_id, 00986 unsigned flags); 00987 00988 H5_DLL herr_t H5C_flush_to_min_clean(H5F_t * f, 00989 hid_t primary_dxpl_id, 00990 hid_t secondary_dxpl_id, 00991 H5C_t * cache_ptr); 00992 00993 H5_DLL herr_t H5C_get_cache_auto_resize_config(const H5C_t * cache_ptr, 00994 H5C_auto_size_ctl_t *config_ptr); 00995 00996 H5_DLL herr_t H5C_get_cache_size(H5C_t * cache_ptr, 00997 size_t * max_size_ptr, 00998 size_t * min_clean_size_ptr, 00999 size_t * cur_size_ptr, 01000 int32_t * cur_num_entries_ptr); 01001 01002 H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t * cache_ptr, 01003 double * hit_rate_ptr); 01004 01005 H5_DLL herr_t H5C_get_entry_status(const H5F_t *f, 01006 haddr_t addr, 01007 size_t * size_ptr, 01008 hbool_t * in_cache_ptr, 01009 hbool_t * is_dirty_ptr, 01010 hbool_t * is_protected_ptr, 01011 hbool_t * is_pinned_ptr); 01012 01013 H5_DLL herr_t H5C_get_evictions_enabled(const H5C_t * cache_ptr, 01014 hbool_t * evictions_enabled_ptr); 01015 01016 H5_DLL herr_t H5C_get_trace_file_ptr(const H5C_t *cache_ptr, 01017 FILE **trace_file_ptr_ptr); 01018 H5_DLL herr_t H5C_get_trace_file_ptr_from_entry(const H5C_cache_entry_t *entry_ptr, 01019 FILE **trace_file_ptr_ptr); 01020 01021 H5_DLL herr_t H5C_insert_entry(H5F_t * f, 01022 hid_t primary_dxpl_id, 01023 hid_t secondary_dxpl_id, 01024 const H5C_class_t * type, 01025 haddr_t addr, 01026 void * thing, 01027 unsigned int flags); 01028 01029 H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t * f, 01030 hid_t primary_dxpl_id, 01031 hid_t secondary_dxpl_id, 01032 H5C_t * cache_ptr, 01033 int32_t ce_array_len, 01034 haddr_t * ce_array_ptr); 01035 01036 H5_DLL herr_t H5C_mark_pinned_entry_dirty(void * thing, 01037 hbool_t size_changed, 01038 size_t new_size); 01039 01040 H5_DLL herr_t H5C_mark_pinned_or_protected_entry_dirty(void *thing); 01041 01042 H5_DLL herr_t H5C_rename_entry(H5C_t * cache_ptr, 01043 const H5C_class_t * type, 01044 haddr_t old_addr, 01045 haddr_t new_addr); 01046 01047 H5_DLL herr_t H5C_pin_protected_entry(void *thing); 01048 01049 H5_DLL void * H5C_protect(H5F_t * f, 01050 hid_t primary_dxpl_id, 01051 hid_t secondary_dxpl_id, 01052 const H5C_class_t * type, 01053 haddr_t addr, 01054 const void * udata1, 01055 void * udata2, 01056 unsigned flags); 01057 01058 H5_DLL herr_t H5C_reset_cache_hit_rate_stats(H5C_t * cache_ptr); 01059 01060 H5_DLL herr_t H5C_resize_pinned_entry(void *thing, size_t new_size); 01061 01062 H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, 01063 H5C_auto_size_ctl_t *config_ptr); 01064 01065 H5_DLL herr_t H5C_set_evictions_enabled(H5C_t *cache_ptr, 01066 hbool_t evictions_enabled); 01067 01068 H5_DLL herr_t H5C_set_prefix(H5C_t * cache_ptr, char * prefix); 01069 01070 H5_DLL herr_t H5C_set_skip_flags(H5C_t * cache_ptr, 01071 hbool_t skip_file_checks, 01072 hbool_t skip_dxpl_id_checks); 01073 01074 H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t * cache_ptr, 01075 FILE * trace_file_ptr); 01076 01077 H5_DLL herr_t H5C_stats(H5C_t * cache_ptr, 01078 const char * cache_name, 01079 hbool_t display_detailed_stats); 01080 01081 H5_DLL void H5C_stats__reset(H5C_t * cache_ptr); 01082 01083 H5_DLL herr_t H5C_unpin_entry(void *thing); 01084 01085 H5_DLL herr_t H5C_unprotect(H5F_t * f, 01086 hid_t primary_dxpl_id, 01087 hid_t secondary_dxpl_id, 01088 const H5C_class_t * type, 01089 haddr_t addr, 01090 void * thing, 01091 unsigned int flags, 01092 size_t new_size); 01093 01094 H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr, 01095 unsigned int tests); 01096 01097 #endif /* !_H5Cprivate_H */ 01098