H5Cprivate.h

00001 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
00002  * Copyright by The HDF Group.                                               *
00003  * Copyright by the Board of Trustees of the University of Illinois.         *
00004  * All rights reserved.                                                      *
00005  *                                                                           *
00006  * This file is part of HDF5.  The full HDF5 copyright notice, including     *
00007  * terms governing use, modification, and redistribution, is contained in    *
00008  * the files COPYING and Copyright.html.  COPYING can be found at the root   *
00009  * of the source code distribution tree; Copyright.html can be found at the  *
00010  * root level of an installed copy of the electronic HDF5 document set and   *
00011  * is linked from the top-level documents page.  It can also be found at     *
00012  * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have          *
00013  * access to either file, you may request a copy from help@hdfgroup.org.     *
00014  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
00015 
00016 /*-------------------------------------------------------------------------
00017  *
00018  * Created:             H5Cprivate.h
00019  *                      6/3/04
00020  *                      John Mainzer
00021  *
00022  * Purpose:             Constants and typedefs available to the rest of the
00023  *                      library.
00024  *
00025  * Modifications:
00026  *
00027  *-------------------------------------------------------------------------
00028  */
00029 
00030 #ifndef _H5Cprivate_H
00031 #define _H5Cprivate_H
00032 
00033 #include "H5Cpublic.h"          /*public prototypes                          */
00034 
00035 /* Pivate headers needed by this header */
00036 #include "H5private.h"          /* Generic Functions                    */
00037 #include "H5Fprivate.h"         /* File access                          */
00038 
00039 
00040 #define H5C_DO_SANITY_CHECKS            0
00041 #define H5C_DO_EXTREME_SANITY_CHECKS    0
00042 
00043 /* This sanity checking constant was picked out of the air.  Increase
00044  * or decrease it if appropriate.  Its purposes is to detect corrupt
00045  * object sizes, so it probably doesn't matter if it is a bit big.
00046  *
00047  *                                      JRM - 5/17/04
00048  */
00049 #define H5C_MAX_ENTRY_SIZE              ((size_t)(32 * 1024 * 1024))
00050 
00051 /* H5C_COLLECT_CACHE_STATS controls overall collection of statistics
00052  * on cache activity.  In general, this #define should be set to 0.
00053  */
00054 #define H5C_COLLECT_CACHE_STATS 0
00055 
00056 /* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics
00057  * in individual cache entries.
00058  *
00059  * H5C_COLLECT_CACHE_ENTRY_STATS should only be defined to true if
00060  * H5C_COLLECT_CACHE_STATS is also defined to true.
00061  */
00062 #if H5C_COLLECT_CACHE_STATS
00063 
00064 #define H5C_COLLECT_CACHE_ENTRY_STATS   1
00065 
00066 #else
00067 
00068 #define H5C_COLLECT_CACHE_ENTRY_STATS   0
00069 
00070 #endif /* H5C_COLLECT_CACHE_STATS */
00071 
00072 
00073 #ifdef H5_HAVE_PARALLEL
00074 
00075 /* we must maintain the clean and dirty LRU lists when we are compiled
00076  * with parallel support.
00077  */
00078 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS  1
00079 
00080 #else /* H5_HAVE_PARALLEL */
00081 
00082 /* The clean and dirty LRU lists don't buy us anything here -- we may
00083  * want them on for testing on occasion, but in general they should be
00084  * off.
00085  */
00086 #define H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS  0
00087 
00088 #endif /* H5_HAVE_PARALLEL */
00089 
00090 
00091 /* Typedef for the main structure for the cache (defined in H5Cpkg.h) */
00092 
00093 typedef struct H5C_t H5C_t;
00094 
00095 
00096 /*
00097  * Class methods pertaining to caching.  Each type of cached object will
00098  * have a constant variable with permanent life-span that describes how
00099  * to cache the object.  That variable will be of type H5C_class_t and
00100  * have the following required fields...
00101  *
00102  * LOAD:        Loads an object from disk to memory.  The function
00103  *              should allocate some data structure and return it.
00104  *
00105  * FLUSH:       Writes some data structure back to disk.  It would be
00106  *              wise for the data structure to include dirty flags to
00107  *              indicate whether it really needs to be written.  This
00108  *              function is also responsible for freeing memory allocated
00109  *              by the LOAD method if the DEST argument is non-zero (by
00110  *              calling the DEST method).
00111  *
00112  * DEST:        Just frees memory allocated by the LOAD method.
00113  *
00114  * CLEAR:       Just marks object as non-dirty.
00115  *
00116  * SIZE:        Report the size (on disk) of the specified cache object.
00117  *              Note that the space allocated on disk may not be contiguous.
00118  */
00119 
00120 #define H5C_CALLBACK__NO_FLAGS_SET              0x0
00121 #define H5C_CALLBACK__SIZE_CHANGED_FLAG         0x1
00122 #define H5C_CALLBACK__RENAMED_FLAG              0x2
00123 
00124 /* Actions that can be reported to 'notify' client callback */
00125 typedef enum H5C_notify_action_t {
00126     H5C_NOTIFY_ACTION_AFTER_INSERT,     /* Entry has been added to the cache */
00127                                         /* (could be loaded from file with
00128                                          *      'protect' call, or inserted
00129                                          *      with 'set' call)
00130                                          */
00131     H5C_NOTIFY_ACTION_BEFORE_EVICT      /* Entry is about to be evicted from cache */
00132 } H5C_notify_action_t;
00133 
00134 typedef void *(*H5C_load_func_t)(H5F_t *f,
00135                                  hid_t dxpl_id,
00136                                  haddr_t addr,
00137                                  const void *udata1,
00138                                  void *udata2);
00139 typedef herr_t (*H5C_flush_func_t)(H5F_t *f,
00140                                    hid_t dxpl_id,
00141                                    hbool_t dest,
00142                                    haddr_t addr,
00143                                    void *thing,
00144                                    unsigned * flags_ptr);
00145 typedef herr_t (*H5C_dest_func_t)(H5F_t *f,
00146                                   void *thing);
00147 typedef herr_t (*H5C_clear_func_t)(H5F_t *f,
00148                                    void *thing,
00149                                    hbool_t dest);
00150 typedef herr_t (*H5C_notify_func_t)(H5C_notify_action_t action,
00151                                  void *thing);
00152 typedef herr_t (*H5C_size_func_t)(const H5F_t *f,
00153                                   const void *thing,
00154                                   size_t *size_ptr);
00155 
00156 typedef struct H5C_class_t {
00157     int                 id;
00158     H5C_load_func_t     load;
00159     H5C_flush_func_t    flush;
00160     H5C_dest_func_t     dest;
00161     H5C_clear_func_t    clear;
00162     H5C_size_func_t     size;
00163 } H5C_class_t;
00164 
00165 
00166 /* Type defintions of call back functions used by the cache as a whole */
00167 
00168 typedef herr_t (*H5C_write_permitted_func_t)(const H5F_t *f,
00169                                              hid_t dxpl_id,
00170                                              hbool_t * write_permitted_ptr);
00171 
00172 typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr,
00173                                        haddr_t addr,
00174                                        hbool_t was_dirty,
00175                                        unsigned flags,
00176                                        int type_id);
00177 
00178 /* Upper and lower limits on cache size.  These limits are picked
00179  * out of a hat -- you should be able to change them as necessary.
00180  *
00181  * However, if you need a very big cache, you should also increase the
00182  * size of the hash table (H5C__HASH_TABLE_LEN in H5Cpkg.h).  The current
00183  * upper bound on cache size is rather large for the current hash table
00184  * size.
00185  */
00186 
00187 #define H5C__MAX_MAX_CACHE_SIZE         ((size_t)(128 * 1024 * 1024))
00188 #define H5C__MIN_MAX_CACHE_SIZE         ((size_t)(1024))
00189 
00190 
00191 /* Default max cache size and min clean size are give here to make
00192  * them generally accessable.
00193  */
00194 
00195 #define H5C__DEFAULT_MAX_CACHE_SIZE     ((size_t)(4 * 1024 * 1024))
00196 #define H5C__DEFAULT_MIN_CLEAN_SIZE     ((size_t)(2 * 1024 * 1024))
00197 
00198 
00199 /****************************************************************************
00200  *
00201  * structure H5C_cache_entry_t
00202  *
00203  * Instances of the H5C_cache_entry_t structure are used to store cache
00204  * entries in a hash table and sometimes in a skip list.
00205  * See H5SL.c for the particulars of the skip list.
00206  *
00207  * In typical application, this structure is the first field in a
00208  * structure to be cached.  For historical reasons, the external module
00209  * is responsible for managing the is_dirty field (this is no longer
00210  * completely true.  See the comment on the is_dirty field for details).
00211  * All other fields are managed by the cache.
00212  *
00213  * The fields of this structure are discussed individually below:
00214  *
00215  *                                              JRM - 4/26/04
00216  *
00217  * magic:       Unsigned 32 bit integer that must always be set to
00218  *              H5C__H5C_CACHE_ENTRY_T_MAGIC when the entry is valid.
00219  *              The field must be set to H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC
00220  *              just before the entry is freed.
00221  *
00222  *              This is necessary, as the LRU list can be changed out
00223  *              from under H5C_make_space_in_cache() by the flush
00224  *              callback which may change the size of an existing entry,
00225  *              and/or load a new entry while serializing the target entry.
00226  *
00227  *              This in turn can cause a recursive call to
00228  *              H5C_make_space_in_cache() which may either flush or evict
00229  *              the next entry that the first invocation of that function
00230  *              was about to examine.
00231  *
00232  *              The magic field allows H5C_make_space_in_cache() to
00233  *              detect this case, and re-start its scan from the bottom
00234  *              of the LRU when this situation occurs.
00235  *
00236  *              This field is only compiled in debug mode.
00237  *
00238  * addr:        Base address of the cache entry on disk.
00239  *
00240  * size:        Length of the cache entry on disk.  Note that unlike normal
00241  *              caches, the entries in this cache are of variable length.
00242  *              The entries should never overlap, and when we do writebacks,
00243  *              we will want to writeback adjacent entries where possible.
00244  *
00245  *              NB: At present, entries need not be contiguous on disk.  Until
00246  *                  we fix this, we can't do much with writing back adjacent
00247  *                  entries.
00248  *
00249  * type:        Pointer to the instance of H5C_class_t containing pointers
00250  *              to the methods for cache entries of the current type.  This
00251  *              field should be NULL when the instance of H5C_cache_entry_t
00252  *              is not in use.
00253  *
00254  *              The name is not particularly descriptive, but is retained
00255  *              to avoid changes in existing code.
00256  *
00257  * is_dirty:    Boolean flag indicating whether the contents of the cache
00258  *              entry has been modified since the last time it was written
00259  *              to disk.
00260  *
00261  *              NOTE: For historical reasons, this field is not maintained
00262  *                    by the cache.  Instead, the module using the cache
00263  *                    sets this flag when it modifies the entry, and the
00264  *                    flush and clear functions supplied by that module
00265  *                    reset the dirty when appropriate.
00266  *
00267  *                    This is a bit quirky, so we may want to change this
00268  *                    someday.  However it will require a change in the
00269  *                    cache interface.
00270  *
00271  *              Update: Management of the is_dirty field has been largely
00272  *                    moved into the cache.  The only remaining exceptions
00273  *                    are the flush and clear functions supplied by the
00274  *                    modules using the cache.  These still clear the
00275  *                    is_dirty field as before.  -- JRM 7/5/05
00276  *
00277  * dirtied:     Boolean flag used to indicate that the entry has been
00278  *              dirtied while protected.
00279  *
00280  *              This field is set to FALSE in the protect call, and may
00281  *              be set to TRUE by the
00282  *              H5C_mark_pinned_or_protected_entry_dirty()
00283  *              call at an time prior to the unprotect call.
00284  *
00285  *              The H5C_mark_pinned_or_protected_entry_dirty() call exists
00286  *              as a convenience function for the fractal heap code which
00287  *              may not know if an entry is protected or pinned, but knows
00288  *              that is either protected or pinned.  The dirtied field was
00289  *              added as in the parallel case, it is necessary to know
00290  *              whether a protected entry was dirty prior to the protect call.
00291  *
00292  * is_protected: Boolean flag indicating whether this entry is protected
00293  *              (or locked, to use more conventional terms).  When it is
00294  *              protected, the entry cannot be flushed or accessed until
00295  *              it is unprotected (or unlocked -- again to use more
00296  *              conventional terms).
00297  *
00298  *              Note that protected entries are removed from the LRU lists
00299  *              and inserted on the protected list.
00300  *
00301  * is_read_only: Boolean flag that is only meaningful if is_protected is
00302  *              TRUE.  In this circumstance, it indicates whether the
00303  *              entry has been protected read only, or read/write.
00304  *
00305  *              If the entry has been protected read only (i.e. is_protected
00306  *              and is_read_only are both TRUE), we allow the entry to be
00307  *              protected more than once.
00308  *
00309  *              In this case, the number of readers is maintained in the
00310  *              ro_ref_count field (see below), and unprotect calls simply
00311  *              decrement that field until it drops to zero, at which point
00312  *              the entry is actually unprotected.
00313  *
00314  * ro_ref_count: Integer field used to maintain a count of the number of
00315  *              outstanding read only protects on this entry.  This field
00316  *              must be zero whenever either is_protected or is_read_only
00317  *              are TRUE.
00318  *
00319  * is_pinned:   Boolean flag indicating whether the entry has been pinned
00320  *              in the cache.
00321  *
00322  *              For very hot entries, the protect / unprotect overhead
00323  *              can become excessive.  Thus the cache has been extended
00324  *              to allow an entry to be "pinned" in the cache.
00325  *
00326  *              Pinning an entry in the cache has several implications:
00327  *
00328  *              1) A pinned entry cannot be evicted.  Thus unprotected
00329  *                 pinned entries must be stored in the pinned entry
00330  *                 list, instead of being managed by the replacement
00331  *                 policy code (LRU at present).
00332  *
00333  *              2) A pinned entry can be accessed or modified at any time.
00334  *                 Therefore, the cache must check with the entry owner
00335  *                 before flushing it.  If permission is denied, the
00336  *                 cache does not flush the entry.
00337  *
00338  *              3) A pinned entry can be marked as dirty (and possibly
00339  *                 change size) while it is unprotected.
00340  *
00341  *              4) The flush-destroy code must allow pinned entries to
00342  *                 be unpinned (and possibly unprotected) during the
00343  *                 flush.
00344  *
00345  *                                                      JRM -- 3/16/06
00346  *
00347  * in_slist:    Boolean flag indicating whether the entry is in the skip list
00348  *              As a general rule, entries are placed in the list when they
00349  *              are marked dirty.  However they may remain in the list after
00350  *              being flushed.
00351  *
00352  *              Update: Dirty entries are now removed from the skip list
00353  *                      when they are flushed.
00354  *
00355  * flush_marker:  Boolean flag indicating that the entry is to be flushed
00356  *              the next time H5C_flush_cache() is called with the
00357  *              H5AC__FLUSH_MARKED_ENTRIES_FLAG.  The flag is reset when
00358  *              the entry is flushed for whatever reason.
00359  *
00360  * clear_on_unprotect:  Boolean flag used only in PHDF5.  When H5C is used
00361  *              to implement the metadata cache In the parallel case, only
00362  *              the cache with mpi rank 0 is allowed to actually write to
00363  *              file -- all other caches must retain dirty entries until they
00364  *              are advised that the entry is clean.
00365  *
00366  *              This flag is used in the case that such an advisory is
00367  *              received when the entry is protected.  If it is set when an
00368  *              entry is unprotected, and the dirtied flag is not set in
00369  *              the unprotect, the entry's is_dirty flag is reset by flushing
00370  *              it with the H5C__FLUSH_CLEAR_ONLY_FLAG.
00371  *
00372  * flush_in_progress:  Boolean flag that is set to true iff the entry
00373  *              is in the process of being flushed.  This allows the cache
00374  *              to detect when a call is the result of a flush callback.
00375  *
00376  * destroy_in_progress:  Boolean flag that is set to true iff the entry
00377  *              is in the process of being flushed and destroyed.
00378  *
00379  * free_file_space_on_destroy:  Boolean flag that is set to true iff the entry
00380  *              is in the process of being flushed and destroyed and the file
00381  *              space used by the object should be freed by the cache client's
00382  *              'dest' callback routine.
00383  *
00384  *
00385  * Fields supporting the hash table:
00386  *
00387  * Fields in the cache are indexed by a more or less conventional hash table.
00388  * If there are multiple entries in any hash bin, they are stored in a doubly
00389  * linked list.
00390  *
00391  * ht_next:     Next pointer used by the hash table to store multiple
00392  *              entries in a single hash bin.  This field points to the
00393  *              next entry in the doubly linked list of entries in the
00394  *              hash bin, or NULL if there is no next entry.
00395  *
00396  * ht_prev:     Prev pointer used by the hash table to store multiple
00397  *              entries in a single hash bin.  This field points to the
00398  *              previous entry in the doubly linked list of entries in
00399  *              the hash bin, or NULL if there is no previuos entry.
00400  *
00401  *
00402  * Fields supporting replacement policies:
00403  *
00404  * The cache must have a replacement policy, and it will usually be
00405  * necessary for this structure to contain fields supporting that policy.
00406  *
00407  * While there has been interest in several replacement policies for
00408  * this cache, the initial development schedule is tight.  Thus I have
00409  * elected to support only a modified LRU policy for the first cut.
00410  *
00411  * When additional replacement policies are added, the fields in this
00412  * section will be used in different ways or not at all.  Thus the
00413  * documentation of these fields is repeated for each replacement policy.
00414  *
00415  * Modified LRU:
00416  *
00417  * When operating in parallel mode, we must ensure that a read does not
00418  * cause a write.  If it does, the process will hang, as the write will
00419  * be collective and the other processes will not know to participate.
00420  *
00421  * To deal with this issue, I have modified the usual LRU policy by adding
00422  * clean and dirty LRU lists to the usual LRU list.  When reading in
00423  * parallel mode, we evict from the clean LRU list only.  This implies
00424  * that we must try to ensure that the clean LRU list is reasonably well
00425  * stocked.  See the comments on H5C_t in H5Cpkg.h for more details.
00426  *
00427  * Note that even if we start with a completely clean cache, a sequence
00428  * of protects without unprotects can empty the clean LRU list.  In this
00429  * case, the cache must grow temporarily.  At the next write, we will
00430  * attempt to evict enough entries to get the cache down to its nominal
00431  * maximum size.
00432  *
00433  * The use of the replacement policy fields under the Modified LRU policy
00434  * is discussed below:
00435  *
00436  * next:        Next pointer in either the LRU or the protected list,
00437  *              depending on the current value of protected.  If there
00438  *              is no next entry on the list, this field should be set
00439  *              to NULL.
00440  *
00441  * prev:        Prev pointer in either the LRU or the protected list,
00442  *              depending on the current value of protected.  If there
00443  *              is no previous entry on the list, this field should be
00444  *              set to NULL.
00445  *
00446  * aux_next:    Next pointer on either the clean or dirty LRU lists.
00447  *              This entry should be NULL when protected is true.  When
00448  *              protected is false, and dirty is true, it should point
00449  *              to the next item on the dirty LRU list.  When protected
00450  *              is false, and dirty is false, it should point to the
00451  *              next item on the clean LRU list.  In either case, when
00452  *              there is no next item, it should be NULL.
00453  *
00454  * aux_prev:    Previous pointer on either the clean or dirty LRU lists.
00455  *              This entry should be NULL when protected is true.  When
00456  *              protected is false, and dirty is true, it should point
00457  *              to the previous item on the dirty LRU list.  When protected
00458  *              is false, and dirty is false, it should point to the
00459  *              previous item on the clean LRU list.  In either case, when
00460  *              there is no previous item, it should be NULL.
00461  *
00462  *
00463  * Cache entry stats collection fields:
00464  *
00465  * These fields should only be compiled in when both H5C_COLLECT_CACHE_STATS
00466  * and H5C_COLLECT_CACHE_ENTRY_STATS are true.  When present, they allow
00467  * collection of statistics on individual cache entries.
00468  *
00469  * accesses:    int32_t containing the number of times this cache entry has
00470  *              been referenced in its lifetime.
00471  *
00472  * clears:      int32_t containing the number of times this cache entry has
00473  *              been cleared in its life time.
00474  *
00475  * flushes:     int32_t containing the number of times this cache entry has
00476  *              been flushed to file in its life time.
00477  *
00478  * pins:        int32_t containing the number of times this cache entry has
00479  *              been pinned in cache in its life time.
00480  *
00481  ****************************************************************************/
00482 
00483 #ifndef NDEBUG
00484 #define H5C__H5C_CACHE_ENTRY_T_MAGIC          0x005CAC0A
00485 #define H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC      0xDeadBeef
00486 #endif /* NDEBUG */
00487 
00488 typedef struct H5C_cache_entry_t
00489 {
00490 #ifndef NDEBUG
00491     uint32_t            magic;
00492 #endif /* NDEBUG */
00493     H5C_t *             cache_ptr;
00494     haddr_t             addr;
00495     size_t              size;
00496     const H5C_class_t * type;
00497     hbool_t             is_dirty;
00498     hbool_t             dirtied;
00499     hbool_t             is_protected;
00500     hbool_t             is_read_only;
00501     int                 ro_ref_count;
00502     hbool_t             is_pinned;
00503     hbool_t             in_slist;
00504     hbool_t             flush_marker;
00505 #ifdef H5_HAVE_PARALLEL
00506     hbool_t             clear_on_unprotect;
00507 #endif /* H5_HAVE_PARALLEL */
00508     hbool_t             flush_in_progress;
00509     hbool_t             destroy_in_progress;
00510     hbool_t             free_file_space_on_destroy;
00511 
00512     /* fields supporting the hash table: */
00513 
00514     struct H5C_cache_entry_t *  ht_next;
00515     struct H5C_cache_entry_t *  ht_prev;
00516 
00517     /* fields supporting replacement policies: */
00518 
00519     struct H5C_cache_entry_t *  next;
00520     struct H5C_cache_entry_t *  prev;
00521     struct H5C_cache_entry_t *  aux_next;
00522     struct H5C_cache_entry_t *  aux_prev;
00523 
00524 #if H5C_COLLECT_CACHE_ENTRY_STATS
00525 
00526     /* cache entry stats fields */
00527 
00528     int32_t                     accesses;
00529     int32_t                     clears;
00530     int32_t                     flushes;
00531     int32_t                     pins;
00532 
00533 #endif /* H5C_COLLECT_CACHE_ENTRY_STATS */
00534 
00535 } H5C_cache_entry_t;
00536 
00537 
00538 /****************************************************************************
00539  *
00540  * structure H5C_auto_size_ctl_t
00541  *
00542  * Instances of H5C_auto_size_ctl_t are used to get and set the control
00543  * fields for automatic cache re-sizing.
00544  *
00545  * The fields of the structure are discussed individually below:
00546  *
00547  * version: Integer field containing the version number of this version
00548  *      of the H5C_auto_size_ctl_t structure.  Any instance of
00549  *      H5C_auto_size_ctl_t passed to the cache must have a known
00550  *      version number, or an error will be flagged.
00551  *
00552  * report_fcn:  Pointer to the function that is to be called to report
00553  *      activities each time the auto cache resize code is executed.  If the
00554  *      field is NULL, no call is made.
00555  *
00556  *      If the field is not NULL, it must contain the address of a function
00557  *      of type H5C_auto_resize_report_fcn.
00558  *
00559  * set_initial_size: Boolean flag indicating whether the size of the
00560  *      initial size of the cache is to be set to the value given in
00561  *      the initial_size field.  If set_initial_size is FALSE, the
00562  *      initial_size field is ignored.
00563  *
00564  * initial_size: If enabled, this field contain the size the cache is
00565  *      to be set to upon receipt of this structure.  Needless to say,
00566  *      initial_size must lie in the closed interval [min_size, max_size].
00567  *
00568  * min_clean_fraction: double in the range 0 to 1 indicating the fraction
00569  *      of the cache that is to be kept clean.  This field is only used
00570  *      in parallel mode.  Typical values are 0.1 to 0.5.
00571  *
00572  * max_size: Maximum size to which the cache can be adjusted.  The
00573  *      supplied value must fall in the closed interval
00574  *      [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE].  Also, max_size must
00575  *      be greater than or equal to min_size.
00576  *
00577  * min_size: Minimum size to which the cache can be adjusted.  The
00578  *      supplied value must fall in the closed interval
00579  *      [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE].  Also, min_size must
00580  *      be less than or equal to max_size.
00581  *
00582  * epoch_length: Number of accesses on the cache over which to collect
00583  *      hit rate stats before running the automatic cache resize code,
00584  *      if it is enabled.
00585  *
00586  *      At the end of an epoch, we discard prior hit rate data and start
00587  *      collecting afresh.  The epoch_length must lie in the closed
00588  *      interval [H5C__MIN_AR_EPOCH_LENGTH, H5C__MAX_AR_EPOCH_LENGTH].
00589  *
00590  *
00591  * Cache size increase control fields:
00592  *
00593  * incr_mode: Instance of the H5C_cache_incr_mode enumerated type whose
00594  *      value indicates how we determine whether the cache size should be
00595  *      increased.  At present there are two possible values:
00596  *
00597  *      H5C_incr__off:  Don't attempt to increase the size of the cache
00598  *              automatically.
00599  *
00600  *              When this increment mode is selected, the remaining fields
00601  *              in the cache size increase section ar ignored.
00602  *
00603  *      H5C_incr__threshold: Attempt to increase the size of the cache
00604  *              whenever the average hit rate over the last epoch drops
00605  *              below the value supplied in the lower_hr_threshold
00606  *              field.
00607  *
00608  *              Note that this attempt will fail if the cache is already
00609  *              at its maximum size, or if the cache is not already using
00610  *              all available space.
00611  *
00612  * lower_hr_threshold: Lower hit rate threshold.  If the increment mode
00613  *      (incr_mode) is H5C_incr__threshold and the hit rate drops below the
00614  *      value supplied in this field in an epoch, increment the cache size by
00615  *      size_increment.  Note that cache size may not be incremented above
00616  *      max_size, and that the increment may be further restricted by the
00617  *      max_increment field if it is enabled.
00618  *
00619  *      When enabled, this field must contain a value in the range [0.0, 1.0].
00620  *      Depending on the incr_mode selected, it may also have to be less than
00621  *      upper_hr_threshold.
00622  *
00623  * increment:  Double containing the multiplier used to derive the new
00624  *      cache size from the old if a cache size increment is triggered.
00625  *      The increment must be greater than 1.0, and should not exceed 2.0.
00626  *
00627  *      The new cache size is obtained by multiplying the current max cache
00628  *      size by the increment, and then clamping to max_size and to stay
00629  *      within the max_increment as necessary.
00630  *
00631  * apply_max_increment:  Boolean flag indicating whether the max_increment
00632  *      field should be used to limit the maximum cache size increment.
00633  *
00634  * max_increment: If enabled by the apply_max_increment field described
00635  *      above, this field contains the maximum number of bytes by which the
00636  *      cache size can be increased in a single re-size.
00637  *
00638  * flash_incr_mode:  Instance of the H5C_cache_flash_incr_mode enumerated
00639  *      type whose value indicates whether and by what algorithm we should
00640  *      make flash increases in the size of the cache to accomodate insertion
00641  *      of large entries and large increases in the size of a single entry.
00642  *
00643  *      The addition of the flash increment mode was occasioned by performance
00644  *      problems that appear when a local heap is increased to a size in excess
00645  *      of the current cache size.  While the existing re-size code dealt with
00646  *      this eventually, performance was very bad for the remainder of the
00647  *      epoch.
00648  *
00649  *      At present, there are two possible values for the flash_incr_mode:
00650  *
00651  *      H5C_flash_incr__off:  Don't perform flash increases in the size of
00652  *              the cache.
00653  *
00654  *      H5C_flash_incr__add_space:  Let x be either the size of a newly
00655  *              newly inserted entry, or the number of bytes by which the
00656  *              size of an existing entry has been increased.
00657  *
00658  *              If
00659  *                      x > flash_threshold * current max cache size,
00660  *
00661  *              increase the current maximum cache size by x * flash_multiple
00662  *              less any free space in the cache, and start a new epoch.  For
00663  *              now at least, pay no attention to the maximum increment.
00664  *
00665  *
00666  *      With a little thought, it should be obvious that the above flash
00667  *      cache size increase algorithm is not sufficient for all circumstances --
00668  *      for example, suppose the user round robins through
00669  *      (1/flash_threshold) +1 groups, adding one data set to each on each
00670  *      pass.  Then all will increase in size at about the same time, requiring
00671  *      the max cache size to at least double to maintain acceptable
00672  *      performance, however the above flash increment algorithm will not be
00673  *      triggered.
00674  *
00675  *      Hopefully, the add space algorithm detailed above will be sufficient
00676  *      for the performance problems encountered to date.  However, we should
00677  *      expect to revisit the issue.
00678  *
00679  * flash_multiple: Double containing the multiple described above in the
00680  *      H5C_flash_incr__add_space section of the discussion of the
00681  *      flash_incr_mode section.  This field is ignored unless flash_incr_mode
00682  *      is H5C_flash_incr__add_space.
00683  *
00684  * flash_threshold: Double containing the factor by which current max cache size
00685  *      is multiplied to obtain the size threshold for the add_space flash
00686  *      increment algorithm.  The field is ignored unless flash_incr_mode is
00687  *      H5C_flash_incr__add_space.
00688  *
00689  *
00690  * Cache size decrease control fields:
00691  *
00692  * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose
00693  *      value indicates how we determine whether the cache size should be
00694  *      decreased.  At present there are four possibilities.
00695  *
00696  *      H5C_decr__off:  Don't attempt to decrease the size of the cache
00697  *              automatically.
00698  *
00699  *              When this increment mode is selected, the remaining fields
00700  *              in the cache size decrease section are ignored.
00701  *
00702  *      H5C_decr__threshold: Attempt to decrease the size of the cache
00703  *              whenever the average hit rate over the last epoch rises
00704  *              above the value supplied in the upper_hr_threshold
00705  *              field.
00706  *
00707  *      H5C_decr__age_out:  At the end of each epoch, search the cache for
00708  *              entries that have not been accessed for at least the number
00709  *              of epochs specified in the epochs_before_eviction field, and
00710  *              evict these entries.  Conceptually, the maximum cache size
00711  *              is then decreased to match the new actual cache size.  However,
00712  *              this reduction may be modified by the min_size, the
00713  *              max_decrement, and/or the empty_reserve.
00714  *
00715  *      H5C_decr__age_out_with_threshold:  Same as age_out, but we only
00716  *              attempt to reduce the cache size when the hit rate observed
00717  *              over the last epoch exceeds the value provided in the
00718  *              upper_hr_threshold field.
00719  *
00720  * upper_hr_threshold: Upper hit rate threshold.  The use of this field
00721  *      varies according to the current decr_mode:
00722  *
00723  *      H5C_decr__off or H5C_decr__age_out:  The value of this field is
00724  *              ignored.
00725  *
00726  *      H5C_decr__threshold:  If the hit rate exceeds this threshold in any
00727  *              epoch, attempt to decrement the cache size by size_decrement.
00728  *
00729  *              Note that cache size may not be decremented below min_size.
00730  *
00731  *              Note also that if the upper_threshold is 1.0, the cache size
00732  *              will never be reduced.
00733  *
00734  *      H5C_decr__age_out_with_threshold:  If the hit rate exceeds this
00735  *              threshold in any epoch, attempt to reduce the cache size
00736  *              by evicting entries that have not been accessed for more
00737  *              than the specified number of epochs.
00738  *
00739  * decrement: This field is only used when the decr_mode is
00740  *      H5C_decr__threshold.
00741  *
00742  *      The field is a double containing the multiplier used to derive the
00743  *      new cache size from the old if a cache size decrement is triggered.
00744  *      The decrement must be in the range 0.0 (in which case the cache will
00745  *      try to contract to its minimum size) to 1.0 (in which case the
00746  *      cache will never shrink).
00747  *
00748  * apply_max_decrement:  Boolean flag used to determine whether decrements
00749  *      in cache size are to be limited by the max_decrement field.
00750  *
00751  * max_decrement: Maximum number of bytes by which the cache size can be
00752  *      decreased in a single re-size.  Note that decrements may also be
00753  *      restricted by the min_size of the cache, and (in age out modes) by
00754  *      the empty_reserve field.
00755  *
00756  * epochs_before_eviction:  Integer field used in H5C_decr__age_out and
00757  *      H5C_decr__age_out_with_threshold decrement modes.
00758  *
00759  *      This field contains the number of epochs an entry must remain
00760  *      unaccessed before it is evicted in an attempt to reduce the
00761  *      cache size.  If applicable, this field must lie in the range
00762  *      [1, H5C__MAX_EPOCH_MARKERS].
00763  *
00764  * apply_empty_reserve:  Boolean field controlling whether the empty_reserve
00765  *      field is to be used in computing the new cache size when the
00766  *      decr_mode is H5C_decr__age_out or H5C_decr__age_out_with_threshold.
00767  *
00768  * empty_reserve:  To avoid a constant racheting down of cache size by small
00769  *      amounts in the H5C_decr__age_out and H5C_decr__age_out_with_threshold
00770  *      modes, this field allows one to require that any cache size
00771  *      reductions leave the specified fraction of unused space in the cache.
00772  *
00773  *      The value of this field must be in the range [0.0, 1.0].  I would
00774  *      expect typical values to be in the range of 0.01 to 0.1.
00775  *
00776  ****************************************************************************/
00777 
00778 #define H5C_RESIZE_CFG__VALIDATE_GENERAL        0x1
00779 #define H5C_RESIZE_CFG__VALIDATE_INCREMENT      0x2
00780 #define H5C_RESIZE_CFG__VALIDATE_DECREMENT      0x4
00781 #define H5C_RESIZE_CFG__VALIDATE_INTERACTIONS   0x8
00782 #define H5C_RESIZE_CFG__VALIDATE_ALL      \
00783 (                                         \
00784     H5C_RESIZE_CFG__VALIDATE_GENERAL |    \
00785     H5C_RESIZE_CFG__VALIDATE_INCREMENT |  \
00786     H5C_RESIZE_CFG__VALIDATE_DECREMENT |  \
00787     H5C_RESIZE_CFG__VALIDATE_INTERACTIONS \
00788 )
00789 
00790 #define H5C__CURR_AUTO_SIZE_CTL_VER             1
00791 #define H5C__CURR_AUTO_RESIZE_RPT_FCN_VER       1
00792 
00793 #define H5C__MAX_EPOCH_MARKERS                  10
00794 
00795 #define H5C__DEF_AR_UPPER_THRESHHOLD            0.9999
00796 #define H5C__DEF_AR_LOWER_THRESHHOLD            0.9
00797 #define H5C__DEF_AR_MAX_SIZE                    ((size_t)(16 * 1024 * 1024))
00798 #define H5C__DEF_AR_INIT_SIZE                   ((size_t)( 1 * 1024 * 1024))
00799 #define H5C__DEF_AR_MIN_SIZE                    ((size_t)( 1 * 1024 * 1024))
00800 #define H5C__DEF_AR_MIN_CLEAN_FRAC              0.5
00801 #define H5C__DEF_AR_INCREMENT                   2.0
00802 #define H5C__DEF_AR_MAX_INCREMENT               ((size_t)( 2 * 1024 * 1024))
00803 #define H5C__DEF_AR_FLASH_MULTIPLE              1.0
00804 #define H5C__DEV_AR_FLASH_THRESHOLD             0.25
00805 #define H5C__DEF_AR_DECREMENT                   0.9
00806 #define H5C__DEF_AR_MAX_DECREMENT               ((size_t)( 1 * 1024 * 1024))
00807 #define H5C__DEF_AR_EPCHS_B4_EVICT              3
00808 #define H5C__DEF_AR_EMPTY_RESERVE               0.05
00809 #define H5C__MIN_AR_EPOCH_LENGTH                100
00810 #define H5C__DEF_AR_EPOCH_LENGTH                50000
00811 #define H5C__MAX_AR_EPOCH_LENGTH                1000000
00812 
00813 enum H5C_resize_status
00814 {
00815     in_spec,
00816     increase,
00817     flash_increase,
00818     decrease,
00819     at_max_size,
00820     at_min_size,
00821     increase_disabled,
00822     decrease_disabled,
00823     not_full
00824 }; /* enum H5C_resize_conditions */
00825 
00826 typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr,
00827                                         int32_t version,
00828                                         double hit_rate,
00829                                         enum H5C_resize_status status,
00830                                         size_t old_max_cache_size,
00831                                         size_t new_max_cache_size,
00832                                         size_t old_min_clean_size,
00833                                         size_t new_min_clean_size);
00834 
00835 typedef struct H5C_auto_size_ctl_t
00836 {
00837     /* general configuration fields: */
00838     int32_t                             version;
00839     H5C_auto_resize_rpt_fcn             rpt_fcn;
00840 
00841     hbool_t                             set_initial_size;
00842     size_t                              initial_size;
00843 
00844     double                              min_clean_fraction;
00845 
00846     size_t                              max_size;
00847     size_t                              min_size;
00848 
00849     int64_t                             epoch_length;
00850 
00851 
00852     /* size increase control fields: */
00853     enum H5C_cache_incr_mode            incr_mode;
00854 
00855     double                              lower_hr_threshold;
00856 
00857     double                              increment;
00858 
00859     hbool_t                             apply_max_increment;
00860     size_t                              max_increment;
00861 
00862     enum H5C_cache_flash_incr_mode      flash_incr_mode;
00863     double                              flash_multiple;
00864     double                              flash_threshold;
00865 
00866     /* size decrease control fields: */
00867     enum H5C_cache_decr_mode            decr_mode;
00868 
00869     double                              upper_hr_threshold;
00870 
00871     double                              decrement;
00872 
00873     hbool_t                             apply_max_decrement;
00874     size_t                              max_decrement;
00875 
00876     int32_t                             epochs_before_eviction;
00877 
00878     hbool_t                             apply_empty_reserve;
00879     double                              empty_reserve;
00880 
00881 } H5C_auto_size_ctl_t;
00882 
00883 
00884 /*
00885  * Library prototypes.
00886  */
00887 
00888 /* #defines of flags used in the flags parameters in some of the
00889  * following function calls.  Note that not all flags are applicable
00890  * to all function calls.  Flags that don't apply to a particular
00891  * function are ignored in that function.
00892  *
00893  * These flags apply to all function calls:
00894  *
00895  *      H5C__NO_FLAGS_SET (generic "no flags set" for all fcn calls)
00896  *
00897  *
00898  * These flags apply to H5C_insert_entry():
00899  *
00900  *      H5C__SET_FLUSH_MARKER_FLAG
00901  *      H5C__PIN_ENTRY_FLAG
00902  *
00903  * These flags apply to H5C_protect()
00904  *
00905  *      H5C__READ_ONLY_FLAG
00906  *
00907  * These flags apply to H5C_unprotect():
00908  *
00909  *      H5C__SET_FLUSH_MARKER_FLAG
00910  *      H5C__DELETED_FLAG
00911  *      H5C__DIRTIED_FLAG
00912  *      H5C__SIZE_CHANGED_FLAG
00913  *      H5C__PIN_ENTRY_FLAG
00914  *      H5C__UNPIN_ENTRY_FLAG
00915  *      H5C__FREE_FILE_SPACE_FLAG
00916  *      H5C__TAKE_OWNERSHIP_FLAG
00917  *
00918  * These flags apply to H5C_expunge_entry():
00919  *
00920  *      H5C__FREE_FILE_SPACE_FLAG
00921  *
00922  * These flags apply to H5C_flush_cache():
00923  *
00924  *      H5C__FLUSH_INVALIDATE_FLAG
00925  *      H5C__FLUSH_CLEAR_ONLY_FLAG
00926  *      H5C__FLUSH_MARKED_ENTRIES_FLAG
00927  *      H5C__FLUSH_IGNORE_PROTECTED_FLAG (can't use this flag in combination
00928  *                                        with H5C__FLUSH_INVALIDATE_FLAG)
00929  *
00930  * These flags apply to H5C_flush_single_entry():
00931  *
00932  *      H5C__FLUSH_INVALIDATE_FLAG
00933  *      H5C__FLUSH_CLEAR_ONLY_FLAG
00934  *      H5C__FLUSH_MARKED_ENTRIES_FLAG
00935  *      H5C__TAKE_OWNERSHIP_FLAG
00936  */
00937 
00938 #define H5C__NO_FLAGS_SET                       0x0000
00939 #define H5C__SET_FLUSH_MARKER_FLAG              0x0001
00940 #define H5C__DELETED_FLAG                       0x0002
00941 #define H5C__DIRTIED_FLAG                       0x0004
00942 #define H5C__SIZE_CHANGED_FLAG                  0x0008
00943 #define H5C__PIN_ENTRY_FLAG                     0x0010
00944 #define H5C__UNPIN_ENTRY_FLAG                   0x0020
00945 #define H5C__FLUSH_INVALIDATE_FLAG              0x0040
00946 #define H5C__FLUSH_CLEAR_ONLY_FLAG              0x0080
00947 #define H5C__FLUSH_MARKED_ENTRIES_FLAG          0x0100
00948 #define H5C__FLUSH_IGNORE_PROTECTED_FLAG        0x0200
00949 #define H5C__READ_ONLY_FLAG                     0x0400
00950 #define H5C__FREE_FILE_SPACE_FLAG               0x0800
00951 #define H5C__TAKE_OWNERSHIP_FLAG                0x1000
00952 
00953 
00954 H5_DLL H5C_t * H5C_create(size_t                     max_cache_size,
00955                           size_t                     min_clean_size,
00956                           int                        max_type_id,
00957                           const char *               (* type_name_table_ptr),
00958                           H5C_write_permitted_func_t check_write_permitted,
00959                           hbool_t                    write_permitted,
00960                           H5C_log_flush_func_t       log_flush,
00961                           void *                     aux_ptr);
00962 
00963 H5_DLL void H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr,
00964                                         int32_t version,
00965                                         double hit_rate,
00966                                         enum H5C_resize_status status,
00967                                         size_t old_max_cache_size,
00968                                         size_t new_max_cache_size,
00969                                         size_t old_min_clean_size,
00970                                         size_t new_min_clean_size);
00971 
00972 H5_DLL herr_t H5C_dest(H5F_t * f,
00973                        hid_t   primary_dxpl_id,
00974                        hid_t   secondary_dxpl_id);
00975 
00976 H5_DLL herr_t H5C_expunge_entry(H5F_t *             f,
00977                                 hid_t               primary_dxpl_id,
00978                                 hid_t               secondary_dxpl_id,
00979                                 const H5C_class_t * type,
00980                                 haddr_t             addr,
00981                                 unsigned            flags);
00982 
00983 H5_DLL herr_t H5C_flush_cache(H5F_t *  f,
00984                               hid_t    primary_dxpl_id,
00985                               hid_t    secondary_dxpl_id,
00986                               unsigned flags);
00987 
00988 H5_DLL herr_t H5C_flush_to_min_clean(H5F_t * f,
00989                                      hid_t   primary_dxpl_id,
00990                                      hid_t   secondary_dxpl_id,
00991                                      H5C_t * cache_ptr);
00992 
00993 H5_DLL herr_t H5C_get_cache_auto_resize_config(const H5C_t * cache_ptr,
00994                                                H5C_auto_size_ctl_t *config_ptr);
00995 
00996 H5_DLL herr_t H5C_get_cache_size(H5C_t * cache_ptr,
00997                                  size_t * max_size_ptr,
00998                                  size_t * min_clean_size_ptr,
00999                                  size_t * cur_size_ptr,
01000                                  int32_t * cur_num_entries_ptr);
01001 
01002 H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t * cache_ptr,
01003                                      double * hit_rate_ptr);
01004 
01005 H5_DLL herr_t H5C_get_entry_status(const H5F_t *f,
01006                                    haddr_t   addr,
01007                                    size_t *  size_ptr,
01008                                    hbool_t * in_cache_ptr,
01009                                    hbool_t * is_dirty_ptr,
01010                                    hbool_t * is_protected_ptr,
01011                                    hbool_t * is_pinned_ptr);
01012 
01013 H5_DLL herr_t H5C_get_evictions_enabled(const H5C_t * cache_ptr,
01014                                         hbool_t * evictions_enabled_ptr);
01015 
01016 H5_DLL herr_t H5C_get_trace_file_ptr(const H5C_t *cache_ptr,
01017     FILE **trace_file_ptr_ptr);
01018 H5_DLL herr_t H5C_get_trace_file_ptr_from_entry(const H5C_cache_entry_t *entry_ptr,
01019     FILE **trace_file_ptr_ptr);
01020 
01021 H5_DLL herr_t H5C_insert_entry(H5F_t *             f,
01022                                hid_t               primary_dxpl_id,
01023                                hid_t               secondary_dxpl_id,
01024                                const H5C_class_t * type,
01025                                haddr_t             addr,
01026                                void *              thing,
01027                                unsigned int        flags);
01028 
01029 H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t   * f,
01030                                         hid_t     primary_dxpl_id,
01031                                         hid_t     secondary_dxpl_id,
01032                                         H5C_t   * cache_ptr,
01033                                         int32_t   ce_array_len,
01034                                         haddr_t * ce_array_ptr);
01035 
01036 H5_DLL herr_t H5C_mark_pinned_entry_dirty(void *  thing,
01037                                           hbool_t size_changed,
01038                                           size_t  new_size);
01039 
01040 H5_DLL herr_t H5C_mark_pinned_or_protected_entry_dirty(void *thing);
01041 
01042 H5_DLL herr_t H5C_rename_entry(H5C_t *             cache_ptr,
01043                                const H5C_class_t * type,
01044                                haddr_t             old_addr,
01045                                haddr_t             new_addr);
01046 
01047 H5_DLL herr_t H5C_pin_protected_entry(void *thing);
01048 
01049 H5_DLL void * H5C_protect(H5F_t *             f,
01050                           hid_t               primary_dxpl_id,
01051                           hid_t               secondary_dxpl_id,
01052                           const H5C_class_t * type,
01053                           haddr_t             addr,
01054                           const void *        udata1,
01055                           void *              udata2,
01056                           unsigned            flags);
01057 
01058 H5_DLL herr_t H5C_reset_cache_hit_rate_stats(H5C_t * cache_ptr);
01059 
01060 H5_DLL herr_t H5C_resize_pinned_entry(void *thing, size_t new_size);
01061 
01062 H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr,
01063                                                H5C_auto_size_ctl_t *config_ptr);
01064 
01065 H5_DLL herr_t H5C_set_evictions_enabled(H5C_t *cache_ptr,
01066                                         hbool_t evictions_enabled);
01067 
01068 H5_DLL herr_t H5C_set_prefix(H5C_t * cache_ptr, char * prefix);
01069 
01070 H5_DLL herr_t H5C_set_skip_flags(H5C_t * cache_ptr,
01071                                  hbool_t skip_file_checks,
01072                                  hbool_t skip_dxpl_id_checks);
01073 
01074 H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t * cache_ptr,
01075                                      FILE * trace_file_ptr);
01076 
01077 H5_DLL herr_t H5C_stats(H5C_t * cache_ptr,
01078                         const char * cache_name,
01079                         hbool_t display_detailed_stats);
01080 
01081 H5_DLL void H5C_stats__reset(H5C_t * cache_ptr);
01082 
01083 H5_DLL herr_t H5C_unpin_entry(void *thing);
01084 
01085 H5_DLL herr_t H5C_unprotect(H5F_t *             f,
01086                             hid_t               primary_dxpl_id,
01087                             hid_t               secondary_dxpl_id,
01088                             const H5C_class_t * type,
01089                             haddr_t             addr,
01090                             void *              thing,
01091                             unsigned int        flags,
01092                             size_t              new_size);
01093 
01094 H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr,
01095                                          unsigned int tests);
01096 
01097 #endif /* !_H5Cprivate_H */
01098