00001 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 00002 * Copyright by The HDF Group. * 00003 * Copyright by the Board of Trustees of the University of Illinois. * 00004 * All rights reserved. * 00005 * * 00006 * This file is part of HDF5. The full HDF5 copyright notice, including * 00007 * terms governing use, modification, and redistribution, is contained in * 00008 * the files COPYING and Copyright.html. COPYING can be found at the root * 00009 * of the source code distribution tree; Copyright.html can be found at the * 00010 * root level of an installed copy of the electronic HDF5 document set and * 00011 * is linked from the top-level documents page. It can also be found at * 00012 * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * 00013 * access to either file, you may request a copy from help@hdfgroup.org. * 00014 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 00015 00016 /*------------------------------------------------------------------------- 00017 * 00018 * Created: H5Cprivate.h 00019 * 6/3/04 00020 * John Mainzer 00021 * 00022 * Purpose: Constants and typedefs available to the rest of the 00023 * library. 00024 * 00025 * Modifications: 00026 * 00027 *------------------------------------------------------------------------- 00028 */ 00029 00030 #ifndef _H5Cprivate_H 00031 #define _H5Cprivate_H 00032 00033 #include "H5Cpublic.h" /*public prototypes */ 00034 00035 /* Pivate headers needed by this header */ 00036 #include "H5private.h" /* Generic Functions */ 00037 #include "H5Fprivate.h" /* File access */ 00038 00039 00040 #define H5C_DO_SANITY_CHECKS 0 00041 #define H5C_DO_EXTREME_SANITY_CHECKS 0 00042 00043 /* This sanity checking constant was picked out of the air. Increase 00044 * or decrease it if appropriate. Its purposes is to detect corrupt 00045 * object sizes, so it probably doesn't matter if it is a bit big. 00046 * 00047 * JRM - 5/17/04 00048 */ 00049 #define H5C_MAX_ENTRY_SIZE ((size_t)(10 * 1024 * 1024)) 00050 00051 /* H5C_COLLECT_CACHE_STATS controls overall collection of statistics 00052 * on cache activity. In general, this #define should be set to 0. 00053 */ 00054 #define H5C_COLLECT_CACHE_STATS 0 00055 00056 /* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics 00057 * in individual cache entries. 00058 * 00059 * H5C_COLLECT_CACHE_ENTRY_STATS should only be defined to true if 00060 * H5C_COLLECT_CACHE_STATS is also defined to true. 00061 */ 00062 #if H5C_COLLECT_CACHE_STATS 00063 00064 #define H5C_COLLECT_CACHE_ENTRY_STATS 1 00065 00066 #else 00067 00068 #define H5C_COLLECT_CACHE_ENTRY_STATS 0 00069 00070 #endif /* H5C_COLLECT_CACHE_STATS */ 00071 00072 00073 #ifdef H5_HAVE_PARALLEL 00074 00075 /* we must maintain the clean and dirty LRU lists when we are compiled 00076 * with parallel support. 00077 */ 00078 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS 1 00079 00080 #else /* H5_HAVE_PARALLEL */ 00081 00082 /* The clean and dirty LRU lists don't buy us anything here -- we may 00083 * want them on for testing on occasion, but in general they should be 00084 * off. 00085 */ 00086 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS 0 00087 00088 #endif /* H5_HAVE_PARALLEL */ 00089 00090 00091 /* Typedef for the main structure for the cache (defined in H5Cpkg.h) */ 00092 00093 typedef struct H5C_t H5C_t; 00094 00095 00096 /* 00097 * Class methods pertaining to caching. Each type of cached object will 00098 * have a constant variable with permanent life-span that describes how 00099 * to cache the object. That variable will be of type H5C_class_t and 00100 * have the following required fields... 00101 * 00102 * LOAD: Loads an object from disk to memory. The function 00103 * should allocate some data structure and return it. 00104 * 00105 * FLUSH: Writes some data structure back to disk. It would be 00106 * wise for the data structure to include dirty flags to 00107 * indicate whether it really needs to be written. This 00108 * function is also responsible for freeing memory allocated 00109 * by the LOAD method if the DEST argument is non-zero (by 00110 * calling the DEST method). 00111 * 00112 * DEST: Just frees memory allocated by the LOAD method. 00113 * 00114 * CLEAR: Just marks object as non-dirty. 00115 * 00116 * SIZE: Report the size (on disk) of the specified cache object. 00117 * Note that the space allocated on disk may not be contiguous. 00118 */ 00119 00120 #define H5C_CALLBACK__NO_FLAGS_SET 0x0 00121 #define H5C_CALLBACK__SIZE_CHANGED_FLAG 0x1 00122 #define H5C_CALLBACK__RENAMED_FLAG 0x2 00123 00124 typedef void *(*H5C_load_func_t)(H5F_t *f, 00125 hid_t dxpl_id, 00126 haddr_t addr, 00127 const void *udata1, 00128 void *udata2); 00129 typedef herr_t (*H5C_flush_func_t)(H5F_t *f, 00130 hid_t dxpl_id, 00131 hbool_t dest, 00132 haddr_t addr, 00133 void *thing, 00134 unsigned * flags_ptr); 00135 typedef herr_t (*H5C_dest_func_t)(H5F_t *f, 00136 void *thing); 00137 typedef herr_t (*H5C_clear_func_t)(H5F_t *f, 00138 void *thing, 00139 hbool_t dest); 00140 typedef herr_t (*H5C_size_func_t)(const H5F_t *f, 00141 const void *thing, 00142 size_t *size_ptr); 00143 00144 typedef struct H5C_class_t { 00145 int id; 00146 H5C_load_func_t load; 00147 H5C_flush_func_t flush; 00148 H5C_dest_func_t dest; 00149 H5C_clear_func_t clear; 00150 H5C_size_func_t size; 00151 } H5C_class_t; 00152 00153 00154 /* Type defintions of call back functions used by the cache as a whole */ 00155 00156 typedef herr_t (*H5C_write_permitted_func_t)(const H5F_t *f, 00157 hid_t dxpl_id, 00158 hbool_t * write_permitted_ptr); 00159 00160 typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr, 00161 haddr_t addr, 00162 hbool_t was_dirty, 00163 unsigned flags, 00164 int type_id); 00165 00166 /* Upper and lower limits on cache size. These limits are picked 00167 * out of a hat -- you should be able to change them as necessary. 00168 * 00169 * However, if you need a very big cache, you should also increase the 00170 * size of the hash table (H5C__HASH_TABLE_LEN in H5Cpkg.h). The current 00171 * upper bound on cache size is rather large for the current hash table 00172 * size. 00173 */ 00174 00175 #define H5C__MAX_MAX_CACHE_SIZE ((size_t)(128 * 1024 * 1024)) 00176 #define H5C__MIN_MAX_CACHE_SIZE ((size_t)(1024)) 00177 00178 00179 /* Default max cache size and min clean size are give here to make 00180 * them generally accessable. 00181 */ 00182 00183 #define H5C__DEFAULT_MAX_CACHE_SIZE ((size_t)(4 * 1024 * 1024)) 00184 #define H5C__DEFAULT_MIN_CLEAN_SIZE ((size_t)(2 * 1024 * 1024)) 00185 00186 00187 /**************************************************************************** 00188 * 00189 * structure H5C_cache_entry_t 00190 * 00191 * Instances of the H5C_cache_entry_t structure are used to store cache 00192 * entries in a hash table and sometimes in a skip list. 00193 * See H5SL.c for the particulars of the skip list. 00194 * 00195 * In typical application, this structure is the first field in a 00196 * structure to be cached. For historical reasons, the external module 00197 * is responsible for managing the is_dirty field (this is no longer 00198 * completely true. See the comment on the is_dirty field for details). 00199 * All other fields are managed by the cache. 00200 * 00201 * The fields of this structure are discussed individually below: 00202 * 00203 * JRM - 4/26/04 00204 * 00205 * magic: Unsigned 32 bit integer that must always be set to 00206 * H5C__H5C_CACHE_ENTRY_T_MAGIC when the entry is valid. 00207 * The field must be set to H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 00208 * just before the entry is freed. 00209 * 00210 * This is necessary, as the LRU list can be changed out 00211 * from under H5C_make_space_in_cache() by the flush 00212 * callback which may change the size of an existing entry, 00213 * and/or load a new entry while serializing the target entry. 00214 * 00215 * This in turn can cause a recursive call to 00216 * H5C_make_space_in_cache() which may either flush or evict 00217 * the next entry that the first invocation of that function 00218 * was about to examine. 00219 * 00220 * The magic field allows H5C_make_space_in_cache() to 00221 * detect this case, and re-start its scan from the bottom 00222 * of the LRU when this situation occurs. 00223 * 00224 * This field is only compiled in debug mode. 00225 * 00226 * addr: Base address of the cache entry on disk. 00227 * 00228 * size: Length of the cache entry on disk. Note that unlike normal 00229 * caches, the entries in this cache are of variable length. 00230 * The entries should never overlap, and when we do writebacks, 00231 * we will want to writeback adjacent entries where possible. 00232 * 00233 * NB: At present, entries need not be contiguous on disk. Until 00234 * we fix this, we can't do much with writing back adjacent 00235 * entries. 00236 * 00237 * type: Pointer to the instance of H5C_class_t containing pointers 00238 * to the methods for cache entries of the current type. This 00239 * field should be NULL when the instance of H5C_cache_entry_t 00240 * is not in use. 00241 * 00242 * The name is not particularly descriptive, but is retained 00243 * to avoid changes in existing code. 00244 * 00245 * is_dirty: Boolean flag indicating whether the contents of the cache 00246 * entry has been modified since the last time it was written 00247 * to disk. 00248 * 00249 * NOTE: For historical reasons, this field is not maintained 00250 * by the cache. Instead, the module using the cache 00251 * sets this flag when it modifies the entry, and the 00252 * flush and clear functions supplied by that module 00253 * reset the dirty when appropriate. 00254 * 00255 * This is a bit quirky, so we may want to change this 00256 * someday. However it will require a change in the 00257 * cache interface. 00258 * 00259 * Update: Management of the is_dirty field has been largely 00260 * moved into the cache. The only remaining exceptions 00261 * are the flush and clear functions supplied by the 00262 * modules using the cache. These still clear the 00263 * is_dirty field as before. -- JRM 7/5/05 00264 * 00265 * dirtied: Boolean flag used to indicate that the entry has been 00266 * dirtied while protected. 00267 * 00268 * This field is set to FALSE in the protect call, and may 00269 * be set to TRUE by the 00270 * H5C_mark_pinned_or_protected_entry_dirty() 00271 * call at an time prior to the unprotect call. 00272 * 00273 * The H5C_mark_pinned_or_protected_entry_dirty() call exists 00274 * as a convenience function for the fractal heap code which 00275 * may not know if an entry is protected or pinned, but knows 00276 * that is either protected or pinned. The dirtied field was 00277 * added as in the parallel case, it is necessary to know 00278 * whether a protected entry was dirty prior to the protect call. 00279 * 00280 * is_protected: Boolean flag indicating whether this entry is protected 00281 * (or locked, to use more conventional terms). When it is 00282 * protected, the entry cannot be flushed or accessed until 00283 * it is unprotected (or unlocked -- again to use more 00284 * conventional terms). 00285 * 00286 * Note that protected entries are removed from the LRU lists 00287 * and inserted on the protected list. 00288 * 00289 * is_read_only: Boolean flag that is only meaningful if is_protected is 00290 * TRUE. In this circumstance, it indicates whether the 00291 * entry has been protected read only, or read/write. 00292 * 00293 * If the entry has been protected read only (i.e. is_protected 00294 * and is_read_only are both TRUE), we allow the entry to be 00295 * protected more than once. 00296 * 00297 * In this case, the number of readers is maintained in the 00298 * ro_ref_count field (see below), and unprotect calls simply 00299 * decrement that field until it drops to zero, at which point 00300 * the entry is actually unprotected. 00301 * 00302 * ro_ref_count: Integer field used to maintain a count of the number of 00303 * outstanding read only protects on this entry. This field 00304 * must be zero whenever either is_protected or is_read_only 00305 * are TRUE. 00306 * 00307 * is_pinned: Boolean flag indicating whether the entry has been pinned 00308 * in the cache. 00309 * 00310 * For very hot entries, the protect / unprotect overhead 00311 * can become excessive. Thus the cache has been extended 00312 * to allow an entry to be "pinned" in the cache. 00313 * 00314 * Pinning an entry in the cache has several implications: 00315 * 00316 * 1) A pinned entry cannot be evicted. Thus unprotected 00317 * pinned entries must be stored in the pinned entry 00318 * list, instead of being managed by the replacement 00319 * policy code (LRU at present). 00320 * 00321 * 2) A pinned entry can be accessed or modified at any time. 00322 * Therefore, the cache must check with the entry owner 00323 * before flushing it. If permission is denied, the 00324 * cache does not flush the entry. 00325 * 00326 * 3) A pinned entry can be marked as dirty (and possibly 00327 * change size) while it is unprotected. 00328 * 00329 * 4) The flush-destroy code must allow pinned entries to 00330 * be unpinned (and possibly unprotected) during the 00331 * flush. 00332 * 00333 * JRM -- 3/16/06 00334 * 00335 * in_slist: Boolean flag indicating whether the entry is in the skip list 00336 * As a general rule, entries are placed in the list when they 00337 * are marked dirty. However they may remain in the list after 00338 * being flushed. 00339 * 00340 * Update: Dirty entries are now removed from the skip list 00341 * when they are flushed. 00342 * 00343 * flush_marker: Boolean flag indicating that the entry is to be flushed 00344 * the next time H5C_flush_cache() is called with the 00345 * H5AC__FLUSH_MARKED_ENTRIES_FLAG. The flag is reset when 00346 * the entry is flushed for whatever reason. 00347 * 00348 * clear_on_unprotect: Boolean flag used only in PHDF5. When H5C is used 00349 * to implement the metadata cache In the parallel case, only 00350 * the cache with mpi rank 0 is allowed to actually write to 00351 * file -- all other caches must retain dirty entries until they 00352 * are advised that the entry is clean. 00353 * 00354 * This flag is used in the case that such an advisory is 00355 * received when the entry is protected. If it is set when an 00356 * entry is unprotected, and the dirtied flag is not set in 00357 * the unprotect, the entry's is_dirty flag is reset by flushing 00358 * it with the H5C__FLUSH_CLEAR_ONLY_FLAG. 00359 * 00360 * flush_in_progress: Boolean flag that is set to true iff the entry 00361 * is in the process of being flushed. This allows the cache 00362 * to detect when a call is the result of a flush callback. 00363 * 00364 * destroy_in_progress: Boolean flag that is set to true iff the entry 00365 * is in the process of being flushed and destroyed. 00366 * 00367 * free_file_space_on_destroy: Boolean flag that is set to true iff the entry 00368 * is in the process of being flushed and destroyed and the file 00369 * space used by the object should be freed by the cache client's 00370 * 'dest' callback routine. 00371 * 00372 * 00373 * Fields supporting the hash table: 00374 * 00375 * Fields in the cache are indexed by a more or less conventional hash table. 00376 * If there are multiple entries in any hash bin, they are stored in a doubly 00377 * linked list. 00378 * 00379 * ht_next: Next pointer used by the hash table to store multiple 00380 * entries in a single hash bin. This field points to the 00381 * next entry in the doubly linked list of entries in the 00382 * hash bin, or NULL if there is no next entry. 00383 * 00384 * ht_prev: Prev pointer used by the hash table to store multiple 00385 * entries in a single hash bin. This field points to the 00386 * previous entry in the doubly linked list of entries in 00387 * the hash bin, or NULL if there is no previuos entry. 00388 * 00389 * 00390 * Fields supporting replacement policies: 00391 * 00392 * The cache must have a replacement policy, and it will usually be 00393 * necessary for this structure to contain fields supporting that policy. 00394 * 00395 * While there has been interest in several replacement policies for 00396 * this cache, the initial development schedule is tight. Thus I have 00397 * elected to support only a modified LRU policy for the first cut. 00398 * 00399 * When additional replacement policies are added, the fields in this 00400 * section will be used in different ways or not at all. Thus the 00401 * documentation of these fields is repeated for each replacement policy. 00402 * 00403 * Modified LRU: 00404 * 00405 * When operating in parallel mode, we must ensure that a read does not 00406 * cause a write. If it does, the process will hang, as the write will 00407 * be collective and the other processes will not know to participate. 00408 * 00409 * To deal with this issue, I have modified the usual LRU policy by adding 00410 * clean and dirty LRU lists to the usual LRU list. When reading in 00411 * parallel mode, we evict from the clean LRU list only. This implies 00412 * that we must try to ensure that the clean LRU list is reasonably well 00413 * stocked. See the comments on H5C_t in H5Cpkg.h for more details. 00414 * 00415 * Note that even if we start with a completely clean cache, a sequence 00416 * of protects without unprotects can empty the clean LRU list. In this 00417 * case, the cache must grow temporarily. At the next write, we will 00418 * attempt to evict enough entries to get the cache down to its nominal 00419 * maximum size. 00420 * 00421 * The use of the replacement policy fields under the Modified LRU policy 00422 * is discussed below: 00423 * 00424 * next: Next pointer in either the LRU or the protected list, 00425 * depending on the current value of protected. If there 00426 * is no next entry on the list, this field should be set 00427 * to NULL. 00428 * 00429 * prev: Prev pointer in either the LRU or the protected list, 00430 * depending on the current value of protected. If there 00431 * is no previous entry on the list, this field should be 00432 * set to NULL. 00433 * 00434 * aux_next: Next pointer on either the clean or dirty LRU lists. 00435 * This entry should be NULL when protected is true. When 00436 * protected is false, and dirty is true, it should point 00437 * to the next item on the dirty LRU list. When protected 00438 * is false, and dirty is false, it should point to the 00439 * next item on the clean LRU list. In either case, when 00440 * there is no next item, it should be NULL. 00441 * 00442 * aux_prev: Previous pointer on either the clean or dirty LRU lists. 00443 * This entry should be NULL when protected is true. When 00444 * protected is false, and dirty is true, it should point 00445 * to the previous item on the dirty LRU list. When protected 00446 * is false, and dirty is false, it should point to the 00447 * previous item on the clean LRU list. In either case, when 00448 * there is no previous item, it should be NULL. 00449 * 00450 * 00451 * Cache entry stats collection fields: 00452 * 00453 * These fields should only be compiled in when both H5C_COLLECT_CACHE_STATS 00454 * and H5C_COLLECT_CACHE_ENTRY_STATS are true. When present, they allow 00455 * collection of statistics on individual cache entries. 00456 * 00457 * accesses: int32_t containing the number of times this cache entry has 00458 * been referenced in its lifetime. 00459 * 00460 * clears: int32_t containing the number of times this cache entry has 00461 * been cleared in its life time. 00462 * 00463 * flushes: int32_t containing the number of times this cache entry has 00464 * been flushed to file in its life time. 00465 * 00466 * pins: int32_t containing the number of times this cache entry has 00467 * been pinned in cache in its life time. 00468 * 00469 ****************************************************************************/ 00470 00471 #ifndef NDEBUG 00472 #define H5C__H5C_CACHE_ENTRY_T_MAGIC 0x005CAC0A 00473 #define H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef 00474 #endif /* NDEBUG */ 00475 00476 typedef struct H5C_cache_entry_t 00477 { 00478 #ifndef NDEBUG 00479 uint32_t magic; 00480 #endif /* NDEBUG */ 00481 haddr_t addr; 00482 size_t size; 00483 const H5C_class_t * type; 00484 hbool_t is_dirty; 00485 hbool_t dirtied; 00486 hbool_t is_protected; 00487 hbool_t is_read_only; 00488 int ro_ref_count; 00489 hbool_t is_pinned; 00490 hbool_t in_slist; 00491 hbool_t flush_marker; 00492 #ifdef H5_HAVE_PARALLEL 00493 hbool_t clear_on_unprotect; 00494 #endif /* H5_HAVE_PARALLEL */ 00495 hbool_t flush_in_progress; 00496 hbool_t destroy_in_progress; 00497 hbool_t free_file_space_on_destroy; 00498 00499 /* fields supporting the hash table: */ 00500 00501 struct H5C_cache_entry_t * ht_next; 00502 struct H5C_cache_entry_t * ht_prev; 00503 00504 /* fields supporting replacement policies: */ 00505 00506 struct H5C_cache_entry_t * next; 00507 struct H5C_cache_entry_t * prev; 00508 struct H5C_cache_entry_t * aux_next; 00509 struct H5C_cache_entry_t * aux_prev; 00510 00511 #if H5C_COLLECT_CACHE_ENTRY_STATS 00512 00513 /* cache entry stats fields */ 00514 00515 int32_t accesses; 00516 int32_t clears; 00517 int32_t flushes; 00518 int32_t pins; 00519 00520 #endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ 00521 00522 } H5C_cache_entry_t; 00523 00524 00525 /**************************************************************************** 00526 * 00527 * structure H5C_auto_size_ctl_t 00528 * 00529 * Instances of H5C_auto_size_ctl_t are used to get and set the control 00530 * fields for automatic cache re-sizing. 00531 * 00532 * The fields of the structure are discussed individually below: 00533 * 00534 * version: Integer field containing the version number of this version 00535 * of the H5C_auto_size_ctl_t structure. Any instance of 00536 * H5C_auto_size_ctl_t passed to the cache must have a known 00537 * version number, or an error will be flagged. 00538 * 00539 * report_fcn: Pointer to the function that is to be called to report 00540 * activities each time the auto cache resize code is executed. If the 00541 * field is NULL, no call is made. 00542 * 00543 * If the field is not NULL, it must contain the address of a function 00544 * of type H5C_auto_resize_report_fcn. 00545 * 00546 * set_initial_size: Boolean flag indicating whether the size of the 00547 * initial size of the cache is to be set to the value given in 00548 * the initial_size field. If set_initial_size is FALSE, the 00549 * initial_size field is ignored. 00550 * 00551 * initial_size: If enabled, this field contain the size the cache is 00552 * to be set to upon receipt of this structure. Needless to say, 00553 * initial_size must lie in the closed interval [min_size, max_size]. 00554 * 00555 * min_clean_fraction: double in the range 0 to 1 indicating the fraction 00556 * of the cache that is to be kept clean. This field is only used 00557 * in parallel mode. Typical values are 0.1 to 0.5. 00558 * 00559 * max_size: Maximum size to which the cache can be adjusted. The 00560 * supplied value must fall in the closed interval 00561 * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, max_size must 00562 * be greater than or equal to min_size. 00563 * 00564 * min_size: Minimum size to which the cache can be adjusted. The 00565 * supplied value must fall in the closed interval 00566 * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, min_size must 00567 * be less than or equal to max_size. 00568 * 00569 * epoch_length: Number of accesses on the cache over which to collect 00570 * hit rate stats before running the automatic cache resize code, 00571 * if it is enabled. 00572 * 00573 * At the end of an epoch, we discard prior hit rate data and start 00574 * collecting afresh. The epoch_length must lie in the closed 00575 * interval [H5C__MIN_AR_EPOCH_LENGTH, H5C__MAX_AR_EPOCH_LENGTH]. 00576 * 00577 * 00578 * Cache size increase control fields: 00579 * 00580 * incr_mode: Instance of the H5C_cache_incr_mode enumerated type whose 00581 * value indicates how we determine whether the cache size should be 00582 * increased. At present there are two possible values: 00583 * 00584 * H5C_incr__off: Don't attempt to increase the size of the cache 00585 * automatically. 00586 * 00587 * When this increment mode is selected, the remaining fields 00588 * in the cache size increase section ar ignored. 00589 * 00590 * H5C_incr__threshold: Attempt to increase the size of the cache 00591 * whenever the average hit rate over the last epoch drops 00592 * below the value supplied in the lower_hr_threshold 00593 * field. 00594 * 00595 * Note that this attempt will fail if the cache is already 00596 * at its maximum size, or if the cache is not already using 00597 * all available space. 00598 * 00599 * lower_hr_threshold: Lower hit rate threshold. If the increment mode 00600 * (incr_mode) is H5C_incr__threshold and the hit rate drops below the 00601 * value supplied in this field in an epoch, increment the cache size by 00602 * size_increment. Note that cache size may not be incremented above 00603 * max_size, and that the increment may be further restricted by the 00604 * max_increment field if it is enabled. 00605 * 00606 * When enabled, this field must contain a value in the range [0.0, 1.0]. 00607 * Depending on the incr_mode selected, it may also have to be less than 00608 * upper_hr_threshold. 00609 * 00610 * increment: Double containing the multiplier used to derive the new 00611 * cache size from the old if a cache size increment is triggered. 00612 * The increment must be greater than 1.0, and should not exceed 2.0. 00613 * 00614 * The new cache size is obtained by multiplying the current max cache 00615 * size by the increment, and then clamping to max_size and to stay 00616 * within the max_increment as necessary. 00617 * 00618 * apply_max_increment: Boolean flag indicating whether the max_increment 00619 * field should be used to limit the maximum cache size increment. 00620 * 00621 * max_increment: If enabled by the apply_max_increment field described 00622 * above, this field contains the maximum number of bytes by which the 00623 * cache size can be increased in a single re-size. 00624 * 00625 * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated 00626 * type whose value indicates whether and by what algorithm we should 00627 * make flash increases in the size of the cache to accomodate insertion 00628 * of large entries and large increases in the size of a single entry. 00629 * 00630 * The addition of the flash increment mode was occasioned by performance 00631 * problems that appear when a local heap is increased to a size in excess 00632 * of the current cache size. While the existing re-size code dealt with 00633 * this eventually, performance was very bad for the remainder of the 00634 * epoch. 00635 * 00636 * At present, there are two possible values for the flash_incr_mode: 00637 * 00638 * H5C_flash_incr__off: Don't perform flash increases in the size of 00639 * the cache. 00640 * 00641 * H5C_flash_incr__add_space: Let x be either the size of a newly 00642 * newly inserted entry, or the number of bytes by which the 00643 * size of an existing entry has been increased. 00644 * 00645 * If 00646 * x > flash_threshold * current max cache size, 00647 * 00648 * increase the current maximum cache size by x * flash_multiple 00649 * less any free space in the cache, and start a new epoch. For 00650 * now at least, pay no attention to the maximum increment. 00651 * 00652 * 00653 * With a little thought, it should be obvious that the above flash 00654 * cache size increase algorithm is not sufficient for all circumstances -- 00655 * for example, suppose the user round robins through 00656 * (1/flash_threshold) +1 groups, adding one data set to each on each 00657 * pass. Then all will increase in size at about the same time, requiring 00658 * the max cache size to at least double to maintain acceptable 00659 * performance, however the above flash increment algorithm will not be 00660 * triggered. 00661 * 00662 * Hopefully, the add space algorithm detailed above will be sufficient 00663 * for the performance problems encountered to date. However, we should 00664 * expect to revisit the issue. 00665 * 00666 * flash_multiple: Double containing the multiple described above in the 00667 * H5C_flash_incr__add_space section of the discussion of the 00668 * flash_incr_mode section. This field is ignored unless flash_incr_mode 00669 * is H5C_flash_incr__add_space. 00670 * 00671 * flash_threshold: Double containing the factor by which current max cache size 00672 * is multiplied to obtain the size threshold for the add_space flash 00673 * increment algorithm. The field is ignored unless flash_incr_mode is 00674 * H5C_flash_incr__add_space. 00675 * 00676 * 00677 * Cache size decrease control fields: 00678 * 00679 * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose 00680 * value indicates how we determine whether the cache size should be 00681 * decreased. At present there are four possibilities. 00682 * 00683 * H5C_decr__off: Don't attempt to decrease the size of the cache 00684 * automatically. 00685 * 00686 * When this increment mode is selected, the remaining fields 00687 * in the cache size decrease section are ignored. 00688 * 00689 * H5C_decr__threshold: Attempt to decrease the size of the cache 00690 * whenever the average hit rate over the last epoch rises 00691 * above the value supplied in the upper_hr_threshold 00692 * field. 00693 * 00694 * H5C_decr__age_out: At the end of each epoch, search the cache for 00695 * entries that have not been accessed for at least the number 00696 * of epochs specified in the epochs_before_eviction field, and 00697 * evict these entries. Conceptually, the maximum cache size 00698 * is then decreased to match the new actual cache size. However, 00699 * this reduction may be modified by the min_size, the 00700 * max_decrement, and/or the empty_reserve. 00701 * 00702 * H5C_decr__age_out_with_threshold: Same as age_out, but we only 00703 * attempt to reduce the cache size when the hit rate observed 00704 * over the last epoch exceeds the value provided in the 00705 * upper_hr_threshold field. 00706 * 00707 * upper_hr_threshold: Upper hit rate threshold. The use of this field 00708 * varies according to the current decr_mode: 00709 * 00710 * H5C_decr__off or H5C_decr__age_out: The value of this field is 00711 * ignored. 00712 * 00713 * H5C_decr__threshold: If the hit rate exceeds this threshold in any 00714 * epoch, attempt to decrement the cache size by size_decrement. 00715 * 00716 * Note that cache size may not be decremented below min_size. 00717 * 00718 * Note also that if the upper_threshold is 1.0, the cache size 00719 * will never be reduced. 00720 * 00721 * H5C_decr__age_out_with_threshold: If the hit rate exceeds this 00722 * threshold in any epoch, attempt to reduce the cache size 00723 * by evicting entries that have not been accessed for more 00724 * than the specified number of epochs. 00725 * 00726 * decrement: This field is only used when the decr_mode is 00727 * H5C_decr__threshold. 00728 * 00729 * The field is a double containing the multiplier used to derive the 00730 * new cache size from the old if a cache size decrement is triggered. 00731 * The decrement must be in the range 0.0 (in which case the cache will 00732 * try to contract to its minimum size) to 1.0 (in which case the 00733 * cache will never shrink). 00734 * 00735 * apply_max_decrement: Boolean flag used to determine whether decrements 00736 * in cache size are to be limited by the max_decrement field. 00737 * 00738 * max_decrement: Maximum number of bytes by which the cache size can be 00739 * decreased in a single re-size. Note that decrements may also be 00740 * restricted by the min_size of the cache, and (in age out modes) by 00741 * the empty_reserve field. 00742 * 00743 * epochs_before_eviction: Integer field used in H5C_decr__age_out and 00744 * H5C_decr__age_out_with_threshold decrement modes. 00745 * 00746 * This field contains the number of epochs an entry must remain 00747 * unaccessed before it is evicted in an attempt to reduce the 00748 * cache size. If applicable, this field must lie in the range 00749 * [1, H5C__MAX_EPOCH_MARKERS]. 00750 * 00751 * apply_empty_reserve: Boolean field controlling whether the empty_reserve 00752 * field is to be used in computing the new cache size when the 00753 * decr_mode is H5C_decr__age_out or H5C_decr__age_out_with_threshold. 00754 * 00755 * empty_reserve: To avoid a constant racheting down of cache size by small 00756 * amounts in the H5C_decr__age_out and H5C_decr__age_out_with_threshold 00757 * modes, this field allows one to require that any cache size 00758 * reductions leave the specified fraction of unused space in the cache. 00759 * 00760 * The value of this field must be in the range [0.0, 1.0]. I would 00761 * expect typical values to be in the range of 0.01 to 0.1. 00762 * 00763 ****************************************************************************/ 00764 00765 #define H5C_RESIZE_CFG__VALIDATE_GENERAL 0x1 00766 #define H5C_RESIZE_CFG__VALIDATE_INCREMENT 0x2 00767 #define H5C_RESIZE_CFG__VALIDATE_DECREMENT 0x4 00768 #define H5C_RESIZE_CFG__VALIDATE_INTERACTIONS 0x8 00769 #define H5C_RESIZE_CFG__VALIDATE_ALL \ 00770 ( \ 00771 H5C_RESIZE_CFG__VALIDATE_GENERAL | \ 00772 H5C_RESIZE_CFG__VALIDATE_INCREMENT | \ 00773 H5C_RESIZE_CFG__VALIDATE_DECREMENT | \ 00774 H5C_RESIZE_CFG__VALIDATE_INTERACTIONS \ 00775 ) 00776 00777 #define H5C__CURR_AUTO_SIZE_CTL_VER 1 00778 #define H5C__CURR_AUTO_RESIZE_RPT_FCN_VER 1 00779 00780 #define H5C__MAX_EPOCH_MARKERS 10 00781 00782 #define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999 00783 #define H5C__DEF_AR_LOWER_THRESHHOLD 0.9 00784 #define H5C__DEF_AR_MAX_SIZE ((size_t)(16 * 1024 * 1024)) 00785 #define H5C__DEF_AR_INIT_SIZE ((size_t)( 1 * 1024 * 1024)) 00786 #define H5C__DEF_AR_MIN_SIZE ((size_t)( 1 * 1024 * 1024)) 00787 #define H5C__DEF_AR_MIN_CLEAN_FRAC 0.5 00788 #define H5C__DEF_AR_INCREMENT 2.0 00789 #define H5C__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024)) 00790 #define H5C__DEF_AR_FLASH_MULTIPLE 1.0 00791 #define H5C__DEV_AR_FLASH_THRESHOLD 0.25 00792 #define H5C__DEF_AR_DECREMENT 0.9 00793 #define H5C__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024)) 00794 #define H5C__DEF_AR_EPCHS_B4_EVICT 3 00795 #define H5C__DEF_AR_EMPTY_RESERVE 0.05 00796 #define H5C__MIN_AR_EPOCH_LENGTH 100 00797 #define H5C__DEF_AR_EPOCH_LENGTH 50000 00798 #define H5C__MAX_AR_EPOCH_LENGTH 1000000 00799 00800 enum H5C_resize_status 00801 { 00802 in_spec, 00803 increase, 00804 flash_increase, 00805 decrease, 00806 at_max_size, 00807 at_min_size, 00808 increase_disabled, 00809 decrease_disabled, 00810 not_full 00811 }; /* enum H5C_resize_conditions */ 00812 00813 typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr, 00814 int32_t version, 00815 double hit_rate, 00816 enum H5C_resize_status status, 00817 size_t old_max_cache_size, 00818 size_t new_max_cache_size, 00819 size_t old_min_clean_size, 00820 size_t new_min_clean_size); 00821 00822 typedef struct H5C_auto_size_ctl_t 00823 { 00824 /* general configuration fields: */ 00825 int32_t version; 00826 H5C_auto_resize_rpt_fcn rpt_fcn; 00827 00828 hbool_t set_initial_size; 00829 size_t initial_size; 00830 00831 double min_clean_fraction; 00832 00833 size_t max_size; 00834 size_t min_size; 00835 00836 int64_t epoch_length; 00837 00838 00839 /* size increase control fields: */ 00840 enum H5C_cache_incr_mode incr_mode; 00841 00842 double lower_hr_threshold; 00843 00844 double increment; 00845 00846 hbool_t apply_max_increment; 00847 size_t max_increment; 00848 00849 enum H5C_cache_flash_incr_mode flash_incr_mode; 00850 double flash_multiple; 00851 double flash_threshold; 00852 00853 /* size decrease control fields: */ 00854 enum H5C_cache_decr_mode decr_mode; 00855 00856 double upper_hr_threshold; 00857 00858 double decrement; 00859 00860 hbool_t apply_max_decrement; 00861 size_t max_decrement; 00862 00863 int32_t epochs_before_eviction; 00864 00865 hbool_t apply_empty_reserve; 00866 double empty_reserve; 00867 00868 } H5C_auto_size_ctl_t; 00869 00870 00871 /* 00872 * Library prototypes. 00873 */ 00874 00875 /* #defines of flags used in the flags parameters in some of the 00876 * following function calls. Note that not all flags are applicable 00877 * to all function calls. Flags that don't apply to a particular 00878 * function are ignored in that function. 00879 * 00880 * These flags apply to all function calls: 00881 * 00882 * H5C__NO_FLAGS_SET (generic "no flags set" for all fcn calls) 00883 * 00884 * 00885 * These flags apply to H5C_insert_entry(): 00886 * 00887 * H5C__SET_FLUSH_MARKER_FLAG 00888 * H5C__PIN_ENTRY_FLAG 00889 * 00890 * These flags apply to H5C_protect() 00891 * 00892 * H5C__READ_ONLY_FLAG 00893 * 00894 * These flags apply to H5C_unprotect(): 00895 * 00896 * H5C__SET_FLUSH_MARKER_FLAG 00897 * H5C__DELETED_FLAG 00898 * H5C__DIRTIED_FLAG 00899 * H5C__SIZE_CHANGED_FLAG 00900 * H5C__PIN_ENTRY_FLAG 00901 * H5C__UNPIN_ENTRY_FLAG 00902 * H5C__FREE_FILE_SPACE_FLAG 00903 * H5C__TAKE_OWNERSHIP_FLAG 00904 * 00905 * These flags apply to H5C_expunge_entry(): 00906 * 00907 * H5C__FREE_FILE_SPACE_FLAG 00908 * 00909 * These flags apply to H5C_flush_cache(): 00910 * 00911 * H5C__FLUSH_INVALIDATE_FLAG 00912 * H5C__FLUSH_CLEAR_ONLY_FLAG 00913 * H5C__FLUSH_MARKED_ENTRIES_FLAG 00914 * H5C__FLUSH_IGNORE_PROTECTED_FLAG (can't use this flag in combination 00915 * with H5C__FLUSH_INVALIDATE_FLAG) 00916 * 00917 * These flags apply to H5C_flush_single_entry(): 00918 * 00919 * H5C__FLUSH_INVALIDATE_FLAG 00920 * H5C__FLUSH_CLEAR_ONLY_FLAG 00921 * H5C__FLUSH_MARKED_ENTRIES_FLAG 00922 * H5C__TAKE_OWNERSHIP_FLAG 00923 */ 00924 00925 #define H5C__NO_FLAGS_SET 0x0000 00926 #define H5C__SET_FLUSH_MARKER_FLAG 0x0001 00927 #define H5C__DELETED_FLAG 0x0002 00928 #define H5C__DIRTIED_FLAG 0x0004 00929 #define H5C__SIZE_CHANGED_FLAG 0x0008 00930 #define H5C__PIN_ENTRY_FLAG 0x0010 00931 #define H5C__UNPIN_ENTRY_FLAG 0x0020 00932 #define H5C__FLUSH_INVALIDATE_FLAG 0x0040 00933 #define H5C__FLUSH_CLEAR_ONLY_FLAG 0x0080 00934 #define H5C__FLUSH_MARKED_ENTRIES_FLAG 0x0100 00935 #define H5C__FLUSH_IGNORE_PROTECTED_FLAG 0x0200 00936 #define H5C__READ_ONLY_FLAG 0x0400 00937 #define H5C__FREE_FILE_SPACE_FLAG 0x0800 00938 #define H5C__TAKE_OWNERSHIP_FLAG 0x1000 00939 00940 00941 H5_DLL H5C_t * H5C_create(size_t max_cache_size, 00942 size_t min_clean_size, 00943 int max_type_id, 00944 const char * (* type_name_table_ptr), 00945 H5C_write_permitted_func_t check_write_permitted, 00946 hbool_t write_permitted, 00947 H5C_log_flush_func_t log_flush, 00948 void * aux_ptr); 00949 00950 H5_DLL void H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr, 00951 int32_t version, 00952 double hit_rate, 00953 enum H5C_resize_status status, 00954 size_t old_max_cache_size, 00955 size_t new_max_cache_size, 00956 size_t old_min_clean_size, 00957 size_t new_min_clean_size); 00958 00959 H5_DLL herr_t H5C_dest(H5F_t * f, 00960 hid_t primary_dxpl_id, 00961 hid_t secondary_dxpl_id, 00962 H5C_t * cache_ptr); 00963 00964 H5_DLL herr_t H5C_dest_empty(H5C_t * cache_ptr); 00965 00966 H5_DLL herr_t H5C_expunge_entry(H5F_t * f, 00967 hid_t primary_dxpl_id, 00968 hid_t secondary_dxpl_id, 00969 H5C_t * cache_ptr, 00970 const H5C_class_t * type, 00971 haddr_t addr, 00972 unsigned flags); 00973 00974 H5_DLL herr_t H5C_flush_cache(H5F_t * f, 00975 hid_t primary_dxpl_id, 00976 hid_t secondary_dxpl_id, 00977 H5C_t * cache_ptr, 00978 unsigned flags); 00979 00980 H5_DLL herr_t H5C_flush_to_min_clean(H5F_t * f, 00981 hid_t primary_dxpl_id, 00982 hid_t secondary_dxpl_id, 00983 H5C_t * cache_ptr); 00984 00985 H5_DLL herr_t H5C_get_cache_auto_resize_config(H5C_t * cache_ptr, 00986 H5C_auto_size_ctl_t *config_ptr); 00987 00988 H5_DLL herr_t H5C_get_cache_size(H5C_t * cache_ptr, 00989 size_t * max_size_ptr, 00990 size_t * min_clean_size_ptr, 00991 size_t * cur_size_ptr, 00992 int32_t * cur_num_entries_ptr); 00993 00994 H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t * cache_ptr, 00995 double * hit_rate_ptr); 00996 00997 H5_DLL herr_t H5C_get_entry_status(H5C_t * cache_ptr, 00998 haddr_t addr, 00999 size_t * size_ptr, 01000 hbool_t * in_cache_ptr, 01001 hbool_t * is_dirty_ptr, 01002 hbool_t * is_protected_ptr, 01003 hbool_t * is_pinned_ptr); 01004 01005 H5_DLL herr_t H5C_get_evictions_enabled(H5C_t * cache_ptr, 01006 hbool_t * evictions_enabled_ptr); 01007 01008 H5_DLL herr_t H5C_get_trace_file_ptr(H5C_t * cache_ptr, 01009 FILE ** trace_file_ptr_ptr); 01010 01011 H5_DLL herr_t H5C_insert_entry(H5F_t * f, 01012 hid_t primary_dxpl_id, 01013 hid_t secondary_dxpl_id, 01014 H5C_t * cache_ptr, 01015 const H5C_class_t * type, 01016 haddr_t addr, 01017 void * thing, 01018 unsigned int flags); 01019 01020 H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t * f, 01021 hid_t primary_dxpl_id, 01022 hid_t secondary_dxpl_id, 01023 H5C_t * cache_ptr, 01024 int32_t ce_array_len, 01025 haddr_t * ce_array_ptr); 01026 01027 H5_DLL herr_t H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, 01028 void * thing, 01029 hbool_t size_changed, 01030 size_t new_size); 01031 01032 H5_DLL herr_t H5C_mark_pinned_or_protected_entry_dirty(H5C_t * cache_ptr, 01033 void * thing); 01034 01035 H5_DLL herr_t H5C_rename_entry(H5C_t * cache_ptr, 01036 const H5C_class_t * type, 01037 haddr_t old_addr, 01038 haddr_t new_addr); 01039 01040 H5_DLL herr_t H5C_pin_protected_entry(H5C_t * cache_ptr, 01041 void * thing); 01042 01043 H5_DLL void * H5C_protect(H5F_t * f, 01044 hid_t primary_dxpl_id, 01045 hid_t secondary_dxpl_id, 01046 H5C_t * cache_ptr, 01047 const H5C_class_t * type, 01048 haddr_t addr, 01049 const void * udata1, 01050 void * udata2, 01051 unsigned flags); 01052 01053 H5_DLL herr_t H5C_reset_cache_hit_rate_stats(H5C_t * cache_ptr); 01054 01055 H5_DLL herr_t H5C_resize_pinned_entry(H5C_t * cache_ptr, 01056 void * thing, 01057 size_t new_size); 01058 01059 H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, 01060 H5C_auto_size_ctl_t *config_ptr); 01061 01062 H5_DLL herr_t H5C_set_evictions_enabled(H5C_t * cache_ptr, 01063 hbool_t evictions_enabled); 01064 01065 H5_DLL herr_t H5C_set_prefix(H5C_t * cache_ptr, char * prefix); 01066 01067 H5_DLL herr_t H5C_set_skip_flags(H5C_t * cache_ptr, 01068 hbool_t skip_file_checks, 01069 hbool_t skip_dxpl_id_checks); 01070 01071 H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t * cache_ptr, 01072 FILE * trace_file_ptr); 01073 01074 H5_DLL herr_t H5C_stats(H5C_t * cache_ptr, 01075 const char * cache_name, 01076 hbool_t display_detailed_stats); 01077 01078 H5_DLL void H5C_stats__reset(H5C_t * cache_ptr); 01079 01080 H5_DLL herr_t H5C_unpin_entry(H5C_t * cache_ptr, void * thing); 01081 01082 H5_DLL herr_t H5C_unprotect(H5F_t * f, 01083 hid_t primary_dxpl_id, 01084 hid_t secondary_dxpl_id, 01085 H5C_t * cache_ptr, 01086 const H5C_class_t * type, 01087 haddr_t addr, 01088 void * thing, 01089 unsigned int flags, 01090 size_t new_size); 01091 01092 H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr, 01093 unsigned int tests); 01094 01095 #endif /* !_H5Cprivate_H */ 01096