H5Dpkg.h

00001 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
00002  * Copyright by The HDF Group.                                               *
00003  * Copyright by the Board of Trustees of the University of Illinois.         *
00004  * All rights reserved.                                                      *
00005  *                                                                           *
00006  * This file is part of HDF5.  The full HDF5 copyright notice, including     *
00007  * terms governing use, modification, and redistribution, is contained in    *
00008  * the files COPYING and Copyright.html.  COPYING can be found at the root   *
00009  * of the source code distribution tree; Copyright.html can be found at the  *
00010  * root level of an installed copy of the electronic HDF5 document set and   *
00011  * is linked from the top-level documents page.  It can also be found at     *
00012  * http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have          *
00013  * access to either file, you may request a copy from help@hdfgroup.org.     *
00014  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
00015 
00016 /*
00017  * Programmer:  Quincey Koziol <koziol@ncsa.uiuc.edu>
00018  *              Monday, April 14, 2003
00019  *
00020  * Purpose:     This file contains declarations which are visible only within
00021  *              the H5D package.  Source files outside the H5D package should
00022  *              include H5Dprivate.h instead.
00023  */
00024 #ifndef H5D_PACKAGE
00025 #error "Do not include this file outside the H5D package!"
00026 #endif
00027 
00028 #ifndef _H5Dpkg_H
00029 #define _H5Dpkg_H
00030 
00031 /* Get package's private header */
00032 #include "H5Dprivate.h"
00033 
00034 /* Other private headers needed by this file */
00035 #include "H5Gprivate.h"         /* Groups                               */
00036 #include "H5SLprivate.h"        /* Skip lists                           */
00037 #include "H5Tprivate.h"         /* Datatypes                            */
00038 
00039 /**************************/
00040 /* Package Private Macros */
00041 /**************************/
00042 
00043 /* The number of reserved IDs in dataset ID group */
00044 #define H5D_RESERVED_ATOMS  0
00045 
00046 /* Set the minimum object header size to create objects with */
00047 #define H5D_MINHDR_SIZE 256
00048 
00049 /* [Simple] Macro to construct a H5D_io_info_t from it's components */
00050 #define H5D_BUILD_IO_INFO_WRT(io_info, ds, dxpl_c, dxpl_i, str, buf)    \
00051     (io_info)->dset = ds;                                               \
00052     (io_info)->dxpl_cache = dxpl_c;                                     \
00053     (io_info)->dxpl_id = dxpl_i;                                        \
00054     (io_info)->store = str;                                             \
00055     (io_info)->op_type = H5D_IO_OP_WRITE;                               \
00056     (io_info)->u.wbuf = buf
00057 #define H5D_BUILD_IO_INFO_RD(io_info, ds, dxpl_c, dxpl_i, str, buf)     \
00058     (io_info)->dset = ds;                                               \
00059     (io_info)->dxpl_cache = dxpl_c;                                     \
00060     (io_info)->dxpl_id = dxpl_i;                                        \
00061     (io_info)->store = str;                                             \
00062     (io_info)->op_type = H5D_IO_OP_READ;                                \
00063     (io_info)->u.rbuf = buf
00064 
00065 #define H5D_CHUNK_HASH(D, ADDR) H5F_addr_hash(ADDR, (D)->cache.chunk.nslots)
00066 
00067 
00068 /****************************/
00069 /* Package Private Typedefs */
00070 /****************************/
00071 
00072 /* Typedef for datatype information for raw data I/O operation */
00073 typedef struct H5D_type_info_t {
00074     /* Initial values */
00075     const H5T_t *mem_type;              /* Pointer to memory datatype */
00076     const H5T_t *dset_type;             /* Pointer to dataset datatype */
00077     H5T_path_t *tpath;                  /* Datatype conversion path */
00078     hid_t src_type_id;                  /* Source datatype ID */
00079     hid_t dst_type_id;                  /* Destination datatype ID */
00080 
00081     /* Computed/derived values */
00082     size_t src_type_size;               /* Size of source type  */
00083     size_t dst_type_size;               /* Size of destination type*/
00084     size_t max_type_size;               /* Size of largest source/destination type */
00085     hbool_t is_conv_noop;               /* Whether the type conversion is a NOOP */
00086     hbool_t is_xform_noop;              /* Whether the data transform is a NOOP */
00087     const H5T_subset_info_t *cmpd_subset;   /* Info related to the compound subset conversion functions */
00088     H5T_bkg_t need_bkg;                 /* Type of background buf needed */
00089     size_t request_nelmts;              /* Requested strip mine */
00090     uint8_t *tconv_buf;                 /* Datatype conv buffer */
00091     hbool_t tconv_buf_allocated;        /* Whether the type conversion buffer was allocated */
00092     uint8_t *bkg_buf;                   /* Background buffer    */
00093     hbool_t bkg_buf_allocated;          /* Whether the background buffer was allocated */
00094 } H5D_type_info_t;
00095 
00096 /* Forward declaration of structs used below */
00097 struct H5D_io_info_t;
00098 struct H5D_chunk_map_t;
00099 
00100 /* Function pointers for I/O on particular types of dataset layouts */
00101 typedef herr_t (*H5D_layout_construct_func_t)(H5F_t *f, H5D_t *dset);
00102 typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
00103     hid_t dapl_id);
00104 typedef hbool_t (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage);
00105 typedef herr_t (*H5D_layout_io_init_func_t)(const struct H5D_io_info_t *io_info,
00106     const H5D_type_info_t *type_info,
00107     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
00108     struct H5D_chunk_map_t *cm);
00109 typedef herr_t (*H5D_layout_read_func_t)(struct H5D_io_info_t *io_info,
00110     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00111     const H5S_t *mem_space, struct H5D_chunk_map_t *fm);
00112 typedef herr_t (*H5D_layout_write_func_t)(struct H5D_io_info_t *io_info,
00113     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00114     const H5S_t *mem_space, struct H5D_chunk_map_t *fm);
00115 typedef ssize_t (*H5D_layout_readvv_func_t)(const struct H5D_io_info_t *io_info,
00116     size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[],
00117     size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
00118 typedef ssize_t (*H5D_layout_writevv_func_t)(const struct H5D_io_info_t *io_info,
00119     size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[],
00120     size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
00121 typedef herr_t (*H5D_layout_flush_func_t)(H5D_t *dataset, hid_t dxpl_id);
00122 typedef herr_t (*H5D_layout_io_term_func_t)(const struct H5D_chunk_map_t *cm);
00123 
00124 /* Typedef for grouping layout I/O routines */
00125 typedef struct H5D_layout_ops_t {
00126     H5D_layout_construct_func_t construct;      /* Layout constructor for new datasets */
00127     H5D_layout_init_func_t init;        /* Layout initializer for dataset */
00128     H5D_layout_is_space_alloc_func_t is_space_alloc;    /* Query routine to determine if storage is allocated */
00129     H5D_layout_io_init_func_t io_init;  /* I/O initialization routine */
00130     H5D_layout_read_func_t ser_read;    /* High-level I/O routine for reading data in serial */
00131     H5D_layout_write_func_t ser_write;  /* High-level I/O routine for writing data in serial */
00132 #ifdef H5_HAVE_PARALLEL
00133     H5D_layout_read_func_t par_read;    /* High-level I/O routine for reading data in parallel */
00134     H5D_layout_write_func_t par_write;  /* High-level I/O routine for writing data in parallel */
00135 #endif /* H5_HAVE_PARALLEL */
00136     H5D_layout_readvv_func_t readvv;    /* Low-level I/O routine for reading data */
00137     H5D_layout_writevv_func_t writevv;  /* Low-level I/O routine for writing data */
00138     H5D_layout_flush_func_t flush;      /* Low-level I/O routine for flushing raw data */
00139     H5D_layout_io_term_func_t io_term;  /* I/O shutdown routine */
00140 } H5D_layout_ops_t;
00141 
00142 /* Function pointers for either multiple or single block I/O access */
00143 typedef herr_t (*H5D_io_single_read_func_t)(const struct H5D_io_info_t *io_info,
00144     const H5D_type_info_t *type_info,
00145     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00146 typedef herr_t (*H5D_io_single_write_func_t)(const struct H5D_io_info_t *io_info,
00147     const H5D_type_info_t *type_info,
00148     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00149 
00150 /* Typedef for raw data I/O framework info */
00151 typedef struct H5D_io_ops_t {
00152     H5D_layout_read_func_t multi_read;          /* High-level I/O routine for reading data */
00153     H5D_layout_write_func_t multi_write;        /* High-level I/O routine for writing data */
00154     H5D_io_single_read_func_t single_read;      /* I/O routine for reading single block */
00155     H5D_io_single_write_func_t single_write;    /* I/O routine for writing single block */
00156 } H5D_io_ops_t;
00157 
00158 /* Typedefs for dataset storage information */
00159 typedef struct {
00160     haddr_t dset_addr;      /* Address of dataset in file */
00161     hsize_t dset_size;      /* Total size of dataset in file */
00162 } H5D_contig_storage_t;
00163 
00164 typedef struct {
00165     hsize_t index;          /* "Index" of chunk in dataset (must be first for TBBT routines) */
00166     hsize_t *offset;        /* Chunk's coordinates in elements */
00167 } H5D_chunk_storage_t;
00168 
00169 typedef struct {
00170     void *buf;              /* Buffer for compact dataset */
00171     hbool_t *dirty;         /* Pointer to dirty flag to mark */
00172 } H5D_compact_storage_t;
00173 
00174 typedef union H5D_storage_t {
00175     H5D_contig_storage_t contig; /* Contiguous information for dataset */
00176     H5D_chunk_storage_t chunk;  /* Chunk information for dataset */
00177     H5D_compact_storage_t compact; /* Compact information for dataset */
00178     H5O_efl_t   efl;            /* External file list information for dataset */
00179 } H5D_storage_t;
00180 
00181 /* Typedef for raw data I/O operation info */
00182 typedef enum H5D_io_op_type_t {
00183     H5D_IO_OP_READ,         /* Read operation */
00184     H5D_IO_OP_WRITE         /* Write operation */
00185 } H5D_io_op_type_t;
00186 
00187 typedef struct H5D_io_info_t {
00188     H5D_t *dset;                /* Pointer to dataset being operated on */
00189 #ifndef H5_HAVE_PARALLEL
00190     const
00191 #endif /* H5_HAVE_PARALLEL */
00192         H5D_dxpl_cache_t *dxpl_cache; /* Pointer to cached DXPL info */
00193     hid_t dxpl_id;              /* Original DXPL ID */
00194 #ifdef H5_HAVE_PARALLEL
00195     MPI_Comm comm;              /* MPI communicator for file */
00196     hbool_t using_mpi_vfd;      /* Whether the file is using an MPI-based VFD */
00197     struct {
00198         H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request (H5D_XFER_IO_XFER_MODE_NAME) */
00199         H5FD_mpio_collective_opt_t coll_opt_mode; /* Parallel transfer with independent IO or collective IO with this mode */
00200         H5D_io_ops_t io_ops;    /* I/O operation function pointers */
00201     } orig;
00202 #endif /* H5_HAVE_PARALLEL */
00203     H5D_storage_t *store;       /* Dataset storage info */
00204     H5D_layout_ops_t layout_ops;    /* Dataset layout I/O operation function pointers */
00205     H5D_io_ops_t io_ops;        /* I/O operation function pointers */
00206     H5D_io_op_type_t op_type;
00207     union {
00208         void *rbuf;             /* Pointer to buffer for read */
00209         const void *wbuf;       /* Pointer to buffer to write */
00210     } u;
00211 } H5D_io_info_t;
00212 
00213 
00214 /******************/
00215 /* Chunk typedefs */
00216 /******************/
00217 
00218 /* Typedef for chunked dataset index operation info */
00219 typedef struct H5D_chk_idx_info_t {
00220     H5F_t *f;                   /* File pointer for operation */
00221     hid_t dxpl_id;              /* DXPL ID for operation */
00222     const H5O_pline_t *pline;   /* I/O pipeline info */
00223     H5O_layout_chunk_t *layout;           /* Chunk layout description */
00224     H5O_storage_chunk_t *storage;         /* Chunk storage description */
00225 } H5D_chk_idx_info_t;
00226 
00227 /*
00228  * "Generic" chunk record.  Each chunk is keyed by the minimum logical
00229  * N-dimensional coordinates and the datatype size of the chunk.
00230  * The fastest-varying dimension is assumed to reference individual bytes of
00231  * the array, so a 100-element 1-D array of 4-byte integers would really be a
00232  * 2-D array with the slow varying dimension of size 100 and the fast varying
00233  * dimension of size 4 (the storage dimensionality has very little to do with
00234  * the real dimensionality).
00235  *
00236  * The chunk's file address, filter mask and size on disk are not key values.
00237  */
00238 typedef struct H5D_chunk_rec_t {
00239     uint32_t    nbytes;                         /* Size of stored data  */
00240     hsize_t     offset[H5O_LAYOUT_NDIMS];       /* Logical offset to start */
00241     unsigned    filter_mask;                    /* Excluded filters     */
00242     haddr_t     chunk_addr;                     /* Address of chunk in file */
00243 } H5D_chunk_rec_t;
00244 
00245 /*
00246  * Common data exchange structure for indexed storage nodes.  This structure is
00247  * passed through the indexing layer to the methods for the objects
00248  * to which the index points.
00249  */
00250 typedef struct H5D_chunk_common_ud_t {
00251     /* downward */
00252     const H5O_layout_chunk_t *layout;           /* Chunk layout description */
00253     const H5O_storage_chunk_t *storage;         /* Chunk storage description */
00254     const hsize_t *offset;                      /* Logical offset of chunk */
00255 } H5D_chunk_common_ud_t;
00256 
00257 /* B-tree callback info for various operations */
00258 typedef struct H5D_chunk_ud_t {
00259     H5D_chunk_common_ud_t common;       /* Common info for B-tree user data (must be first) */
00260 
00261     /* Upward */
00262     uint32_t    nbytes;                 /*size of stored data   */
00263     unsigned    filter_mask;            /*excluded filters      */
00264     haddr_t     addr;                   /*file address of chunk */
00265 } H5D_chunk_ud_t;
00266 
00267 /* Typedef for "generic" chunk callbacks */
00268 typedef int (*H5D_chunk_cb_func_t)(const H5D_chunk_rec_t *chunk_rec,
00269     void *udata);
00270 
00271 /* Typedefs for chunk operations */
00272 typedef herr_t (*H5D_chunk_init_func_t)(const H5D_chk_idx_info_t *idx_info,
00273     const H5S_t *space, haddr_t dset_ohdr_addr);
00274 typedef herr_t (*H5D_chunk_create_func_t)(const H5D_chk_idx_info_t *idx_info);
00275 typedef hbool_t (*H5D_chunk_is_space_alloc_func_t)(const H5O_storage_chunk_t *storage);
00276 typedef herr_t (*H5D_chunk_insert_func_t)(const H5D_chk_idx_info_t *idx_info,
00277     H5D_chunk_ud_t *udata);
00278 typedef herr_t (*H5D_chunk_get_addr_func_t)(const H5D_chk_idx_info_t *idx_info,
00279     H5D_chunk_ud_t *udata);
00280 typedef herr_t (*H5D_chunk_resize_func_t)(H5O_layout_chunk_t *layout);
00281 typedef int (*H5D_chunk_iterate_func_t)(const H5D_chk_idx_info_t *idx_info,
00282     H5D_chunk_cb_func_t chunk_cb, void *chunk_udata);
00283 typedef herr_t (*H5D_chunk_remove_func_t)(const H5D_chk_idx_info_t *idx_info,
00284     H5D_chunk_common_ud_t *udata);
00285 typedef herr_t (*H5D_chunk_delete_func_t)(const H5D_chk_idx_info_t *idx_info);
00286 typedef herr_t (*H5D_chunk_copy_setup_func_t)(const H5D_chk_idx_info_t *idx_info_src,
00287     const H5D_chk_idx_info_t *idx_info_dst);
00288 typedef herr_t (*H5D_chunk_copy_shutdown_func_t)(H5O_storage_chunk_t *storage_src,
00289     H5O_storage_chunk_t *storage_dst, hid_t dxpl_id);
00290 typedef herr_t (*H5D_chunk_size_func_t)(const H5D_chk_idx_info_t *idx_info,
00291     hsize_t *idx_size);
00292 typedef herr_t (*H5D_chunk_reset_func_t)(H5O_storage_chunk_t *storage, hbool_t reset_addr);
00293 typedef herr_t (*H5D_chunk_dump_func_t)(const H5O_storage_chunk_t *storage,
00294     FILE *stream);
00295 typedef herr_t (*H5D_chunk_dest_func_t)(const H5D_chk_idx_info_t *idx_info);
00296 
00297 /* Typedef for grouping chunk I/O routines */
00298 typedef struct H5D_chunk_ops_t {
00299     H5D_chunk_init_func_t init;             /* Routine to initialize indexing information in memory */
00300     H5D_chunk_create_func_t create;         /* Routine to create chunk index */
00301     H5D_chunk_is_space_alloc_func_t is_space_alloc;    /* Query routine to determine if storage/index is allocated */
00302     H5D_chunk_insert_func_t insert;         /* Routine to insert a chunk into an index */
00303     H5D_chunk_get_addr_func_t get_addr;     /* Routine to retrieve address of chunk in file */
00304     H5D_chunk_resize_func_t resize;         /* Routine to update chunk index info after resizing dataset */
00305     H5D_chunk_iterate_func_t iterate;       /* Routine to iterate over chunks */
00306     H5D_chunk_remove_func_t remove;         /* Routine to remove a chunk from an index */
00307     H5D_chunk_delete_func_t idx_delete;     /* Routine to delete index & all chunks from file*/
00308     H5D_chunk_copy_setup_func_t copy_setup; /* Routine to perform any necessary setup for copying chunks */
00309     H5D_chunk_copy_shutdown_func_t copy_shutdown; /* Routine to perform any necessary shutdown for copying chunks */
00310     H5D_chunk_size_func_t size;             /* Routine to get size of indexing information */
00311     H5D_chunk_reset_func_t reset;           /* Routine to reset indexing information */
00312     H5D_chunk_dump_func_t dump;             /* Routine to dump indexing information */
00313     H5D_chunk_dest_func_t dest;             /* Routine to destroy indexing information in memory */
00314 } H5D_chunk_ops_t;
00315 
00316 /* Structure holding information about a chunk's selection for mapping */
00317 typedef struct H5D_chunk_info_t {
00318     hsize_t index;              /* "Index" of chunk in dataset */
00319     uint32_t chunk_points;      /* Number of elements selected in chunk */
00320     hsize_t coords[H5O_LAYOUT_NDIMS];   /* Coordinates of chunk in file dataset's dataspace */
00321     H5S_t *fspace;              /* Dataspace describing chunk & selection in it */
00322     unsigned fspace_shared;     /* Indicate that the file space for a chunk is shared and shouldn't be freed */
00323     H5S_t *mspace;              /* Dataspace describing selection in memory corresponding to this chunk */
00324     unsigned mspace_shared;     /* Indicate that the memory space for a chunk is shared and shouldn't be freed */
00325 } H5D_chunk_info_t;
00326 
00327 /* Main structure holding the mapping between file chunks and memory */
00328 typedef struct H5D_chunk_map_t {
00329     H5O_layout_t *layout;       /* Dataset layout information*/
00330     hsize_t nelmts;             /* Number of elements selected in file & memory dataspaces */
00331 
00332     const H5S_t *file_space;    /* Pointer to the file dataspace */
00333     unsigned f_ndims;           /* Number of dimensions for file dataspace */
00334     hsize_t f_dims[H5O_LAYOUT_NDIMS];   /* File dataspace dimensions */
00335 
00336     const H5S_t *mem_space;     /* Pointer to the memory dataspace */
00337     H5S_t *mchunk_tmpl;         /* Dataspace template for new memory chunks */
00338     H5S_sel_iter_t mem_iter;    /* Iterator for elements in memory selection */
00339     unsigned m_ndims;           /* Number of dimensions for memory dataspace */
00340     H5S_sel_type msel_type;     /* Selection type in memory */
00341 
00342     H5SL_t *sel_chunks;         /* Skip list containing information for each chunk selected */
00343 
00344     H5S_t  *single_space;       /* Dataspace for single chunk */
00345     H5D_chunk_info_t *single_chunk_info;  /* Pointer to single chunk's info */
00346     hbool_t use_single;         /* Whether I/O is on a single element */
00347 
00348     hsize_t last_index;         /* Index of last chunk operated on */
00349     H5D_chunk_info_t *last_chunk_info;  /* Pointer to last chunk's info */
00350 
00351     hsize_t chunk_dim[H5O_LAYOUT_NDIMS];    /* Size of chunk in each dimension */
00352 
00353 #ifdef H5_HAVE_PARALLEL
00354     H5D_chunk_info_t **select_chunk;    /* Store the information about whether this chunk is selected or not */
00355 #endif /* H5_HAVE_PARALLEL */
00356 } H5D_chunk_map_t;
00357 
00358 /* Cached information about a particular chunk */
00359 typedef struct H5D_chunk_cached_t{
00360     hbool_t     valid;                          /*whether cache info is valid*/
00361     hsize_t     offset[H5O_LAYOUT_NDIMS];       /*logical offset to start*/
00362     uint32_t    nbytes;                         /*size of stored data   */
00363     unsigned    filter_mask;                    /*excluded filters      */
00364     haddr_t     addr;                           /*file address of chunk */
00365 } H5D_chunk_cached_t;
00366 
00367 /* The raw data chunk cache */
00368 typedef struct H5D_rdcc_t {
00369     struct {
00370         unsigned        ninits; /* Number of chunk creations            */
00371         unsigned        nhits;  /* Number of cache hits                 */
00372         unsigned        nmisses;/* Number of cache misses               */
00373         unsigned        nflushes;/* Number of cache flushes             */
00374     } stats;
00375     size_t              nbytes_max; /* Maximum cached raw data in bytes */
00376     size_t              nslots; /* Number of chunk slots allocated      */
00377     double              w0;     /* Chunk preemption policy          */
00378     struct H5D_rdcc_ent_t *head; /* Head of doubly linked list          */
00379     struct H5D_rdcc_ent_t *tail; /* Tail of doubly linked list          */
00380     size_t              nbytes_used; /* Current cached raw data in bytes */
00381     int                 nused;  /* Number of chunk slots in use         */
00382     H5D_chunk_cached_t last;    /* Cached copy of last chunk information */
00383     struct H5D_rdcc_ent_t **slot; /* Chunk slots, each points to a chunk*/
00384     H5SL_t              *sel_chunks; /* Skip list containing information for each chunk selected */
00385     H5S_t               *single_space; /* Dataspace for single element I/O on chunks */
00386     H5D_chunk_info_t *single_chunk_info;  /* Pointer to single chunk's info */
00387 } H5D_rdcc_t;
00388 
00389 /* The raw data contiguous data cache */
00390 typedef struct H5D_rdcdc_t {
00391     unsigned char *sieve_buf;   /* Buffer to hold data sieve buffer */
00392     haddr_t sieve_loc;          /* File location (offset) of the data sieve buffer */
00393     size_t sieve_size;          /* Size of the data sieve buffer used (in bytes) */
00394     size_t sieve_buf_size;      /* Size of the data sieve buffer allocated (in bytes) */
00395     unsigned sieve_dirty;       /* Flag to indicate that the data sieve buffer is dirty */
00396 } H5D_rdcdc_t;
00397 
00398 /*
00399  * A dataset is made of two layers, an H5D_t struct that is unique to
00400  * each instance of an opened datset, and a shared struct that is only
00401  * created once for a given dataset.  Thus, if a dataset is opened twice,
00402  * there will be two IDs and two H5D_t structs, both sharing one H5D_shared_t.
00403  */
00404 typedef struct H5D_shared_t {
00405     size_t              fo_count;       /* Reference count */
00406     hid_t               type_id;        /* ID for dataset's datatype    */
00407     H5T_t              *type;           /* Datatype for this dataset     */
00408     H5S_t              *space;          /* Dataspace of this dataset    */
00409     hbool_t             space_dirty;    /* Whether the dataspace info needs to be flushed to the file */
00410     hbool_t             layout_dirty;   /* Whether the layout info needs to be flushed to the file */
00411     hid_t               dcpl_id;        /* Dataset creation property id */
00412     H5D_dcpl_cache_t    dcpl_cache;     /* Cached DCPL values */
00413     H5O_layout_t        layout;         /* Data layout                  */
00414     hbool_t             checked_filters;/* TRUE if dataset passes can_apply check */
00415 
00416     /* Buffered/cached information for types of raw data storage*/
00417     struct {
00418         H5D_rdcdc_t     contig;         /* Information about contiguous data */
00419                                         /* (Note that the "contig" cache
00420                                          * information can be used by a chunked
00421                                          * dataset in certain circumstances)
00422                                          */
00423         H5D_rdcc_t      chunk;          /* Information about chunked data */
00424     } cache;
00425 } H5D_shared_t;
00426 
00427 struct H5D_t {
00428     H5O_loc_t           oloc;           /* Object header location       */
00429     H5G_name_t          path;           /* Group hierarchy path         */
00430     H5D_shared_t        *shared;        /* cached information from file */
00431 };
00432 
00433 /* Enumerated type for allocating dataset's storage */
00434 typedef enum {
00435     H5D_ALLOC_CREATE,           /* Dataset is being created */
00436     H5D_ALLOC_OPEN,             /* Dataset is being opened */
00437     H5D_ALLOC_EXTEND,           /* Dataset's dataspace is being extended */
00438     H5D_ALLOC_WRITE             /* Dataset is being extended */
00439 } H5D_time_alloc_t;
00440 
00441 
00442 /* Typedef for dataset creation operation */
00443 typedef struct {
00444     hid_t type_id;              /* Datatype for dataset */
00445     const H5S_t *space;         /* Dataspace for dataset */
00446     hid_t dcpl_id;              /* Dataset creation property list */
00447     hid_t dapl_id;              /* Dataset access property list */
00448 } H5D_obj_create_t;
00449 
00450 /* Typedef for filling a buffer with a fill value */
00451 typedef struct H5D_fill_buf_info_t {
00452     hbool_t     alloc_vl_during_refill; /* Whether to allocate VL-datatype fill buffer during refill */
00453     H5MM_allocate_t fill_alloc_func;    /* Routine to call for allocating fill buffer */
00454     void        *fill_alloc_info;       /* Extra info for allocation routine */
00455     H5MM_free_t fill_free_func;         /* Routine to call for freeing fill buffer */
00456     void        *fill_free_info;        /* Extra info for free routine */
00457     H5T_path_t *fill_to_mem_tpath;      /* Datatype conversion path for converting the fill value to the memory buffer */
00458     H5T_path_t *mem_to_dset_tpath;      /* Datatype conversion path for converting the memory buffer to the dataset elements */
00459     const H5O_fill_t *fill;             /* Pointer to fill value */
00460     void       *fill_buf;               /* Fill buffer */
00461     size_t      fill_buf_size;          /* Size of fill buffer */
00462     hbool_t     use_caller_fill_buf;    /* Whether the caller provided the fill buffer */
00463     void       *bkg_buf;                /* Background conversion buffer */
00464     size_t      bkg_buf_size;           /* Size of background buffer */
00465     H5T_t      *mem_type;               /* Pointer to memory datatype */
00466     const H5T_t *file_type;             /* Pointer to file datatype */
00467     hid_t       mem_tid;                /* ID for memory version of disk datatype */
00468     hid_t       file_tid;               /* ID for disk datatype */
00469     size_t      mem_elmt_size, file_elmt_size;       /* Size of element in memory and on disk */
00470     size_t      max_elmt_size;          /* Max. size of memory or file datatype */
00471     size_t      elmts_per_buf;          /* # of elements that fit into a buffer */
00472     hbool_t     has_vlen_fill_type;     /* Whether the datatype for the fill value has a variable-length component */
00473 } H5D_fill_buf_info_t;
00474 
00475 /* Internal data structure for computing variable-length dataset's total size */
00476 typedef struct {
00477     hid_t dataset_id;   /* ID of the dataset we are working on */
00478     hid_t fspace_id;    /* ID of the file dataset's dataspace we are working on */
00479     hid_t mspace_id;    /* ID of the memory dataset's dataspace we are working on */
00480     void *fl_tbuf;      /* Ptr to the temporary buffer we are using for fixed-length data */
00481     void *vl_tbuf;      /* Ptr to the temporary buffer we are using for VL data */
00482     hid_t xfer_pid;     /* ID of the dataset xfer property list */
00483     hsize_t size;       /* Accumulated number of bytes for the selection */
00484 } H5D_vlen_bufsize_t;
00485 
00486 /* Raw data chunks are cached.  Each entry in the cache is: */
00487 typedef struct H5D_rdcc_ent_t {
00488     hbool_t     locked;         /*entry is locked in cache              */
00489     hbool_t     dirty;          /*needs to be written to disk?          */
00490     hsize_t     offset[H5O_LAYOUT_NDIMS]; /*chunk name                  */
00491     uint32_t    rd_count;       /*bytes remaining to be read            */
00492     uint32_t    wr_count;       /*bytes remaining to be written         */
00493     haddr_t     chunk_addr;     /*address of chunk in file              */
00494     uint8_t     *chunk;         /*the unfiltered chunk data             */
00495     unsigned    idx;            /*index in hash table                   */
00496     struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list       */
00497     struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list   */
00498 } H5D_rdcc_ent_t;
00499 typedef H5D_rdcc_ent_t *H5D_rdcc_ent_ptr_t; /* For free lists */
00500 
00501 
00502 /*****************************/
00503 /* Package Private Variables */
00504 /*****************************/
00505 extern H5D_dxpl_cache_t H5D_def_dxpl_cache;
00506 
00507 /* Storage layout class I/O operations */
00508 H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_CONTIG[1];
00509 H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_EFL[1];
00510 H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_COMPACT[1];
00511 H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_CHUNK[1];
00512 
00513 /* Chunked layout operations */
00514 H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BTREE[1];
00515 
00516 
00517 /******************************/
00518 /* Package Private Prototypes */
00519 /******************************/
00520 
00521 H5_DLL H5D_t *H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space,
00522     hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id);
00523 H5_DLL H5D_t *H5D_create_named(const H5G_loc_t *loc, const char *name,
00524     hid_t type_id, const H5S_t *space, hid_t lcpl_id, hid_t dcpl_id,
00525     hid_t dapl_id, hid_t dxpl_id);
00526 H5_DLL herr_t H5D_get_space_status(H5D_t *dset, H5D_space_status_t *allocation,
00527     hid_t dxpl_id);
00528 H5_DLL herr_t H5D_alloc_storage(H5D_t *dset, hid_t dxpl_id, H5D_time_alloc_t time_alloc,
00529     hbool_t full_overwrite, hsize_t old_dim[]);
00530 H5_DLL hsize_t H5D_get_storage_size(H5D_t *dset, hid_t dxpl_id);
00531 H5_DLL haddr_t H5D_get_offset(const H5D_t *dset);
00532 H5_DLL herr_t H5D_iterate(void *buf, hid_t type_id, const H5S_t *space,
00533     H5D_operator_t op, void *operator_data);
00534 H5_DLL void * H5D_vlen_get_buf_size_alloc(size_t size, void *info);
00535 H5_DLL herr_t H5D_vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim,
00536     const hsize_t *point, void *op_data);
00537 H5_DLL herr_t H5D_check_filters(H5D_t *dataset);
00538 H5_DLL herr_t H5D_set_extent(H5D_t *dataset, const hsize_t *size, hid_t dxpl_id);
00539 H5_DLL herr_t H5D_get_dxpl_cache(hid_t dxpl_id, H5D_dxpl_cache_t **cache);
00540 H5_DLL herr_t H5D_flush_sieve_buf(H5D_t *dataset, hid_t dxpl_id);
00541 
00542 /* Functions that perform direct serial I/O operations */
00543 H5_DLL herr_t H5D_select_read(const H5D_io_info_t *io_info,
00544     const H5D_type_info_t *type_info,
00545     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00546 H5_DLL herr_t H5D_select_write(const H5D_io_info_t *io_info,
00547     const H5D_type_info_t *type_info,
00548     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00549 
00550 /* Functions that perform scatter-gather serial I/O operations */
00551 H5_DLL herr_t H5D_scatter_mem(const void *_tscat_buf,
00552     const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts,
00553     const H5D_dxpl_cache_t *dxpl_cache, void *_buf);
00554 H5_DLL herr_t H5D_scatgath_read(const H5D_io_info_t *io_info,
00555     const H5D_type_info_t *type_info,
00556     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00557 H5_DLL herr_t H5D_scatgath_write(const H5D_io_info_t *io_info,
00558     const H5D_type_info_t *type_info,
00559     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00560 
00561 /* Functions that operate on dataset's layout information */
00562 H5_DLL herr_t H5D_layout_set_io_ops(const H5D_t *dataset);
00563 H5_DLL size_t H5D_layout_meta_size(const H5F_t *f, const H5O_layout_t *layout,
00564     hbool_t include_compact_data);
00565 H5_DLL herr_t H5D_layout_oh_create(H5F_t *file, hid_t dxpl_id, H5O_t *oh,
00566     H5D_t *dset, hid_t dapl_id);
00567 H5_DLL herr_t H5D_layout_oh_read(H5D_t *dset, hid_t dxpl_id, hid_t dapl_id,
00568     H5P_genplist_t *plist);
00569 H5_DLL herr_t H5D_layout_oh_write(H5D_t *dataset, hid_t dxpl_id, H5O_t *oh,
00570     unsigned update_flags);
00571 
00572 /* Functions that operate on contiguous storage */
00573 H5_DLL herr_t H5D_contig_alloc(H5F_t *f, hid_t dxpl_id,
00574     H5O_storage_contig_t *storage);
00575 H5_DLL hbool_t H5D_contig_is_space_alloc(const H5O_storage_t *storage);
00576 H5_DLL herr_t H5D_contig_fill(H5D_t *dset, hid_t dxpl_id);
00577 H5_DLL haddr_t H5D_contig_get_addr(const H5D_t *dset);
00578 H5_DLL herr_t H5D_contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
00579     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
00580     H5D_chunk_map_t *fm);
00581 H5_DLL herr_t H5D_contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
00582     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space,
00583     H5D_chunk_map_t *fm);
00584 H5_DLL ssize_t H5D_contig_readvv(const H5D_io_info_t *io_info,
00585     size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[],
00586     size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
00587 H5_DLL ssize_t H5D_contig_writevv(const H5D_io_info_t *io_info,
00588     size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[],
00589     size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]);
00590 H5_DLL herr_t H5D_contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src,
00591     H5F_t *f_dst, H5O_storage_contig_t *storage_dst, H5T_t *src_dtype,
00592     H5O_copy_t *cpy_info, hid_t dxpl_id);
00593 
00594 /* Functions that operate on chunked dataset storage */
00595 H5_DLL htri_t H5D_chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr,
00596     hbool_t write_op);
00597 H5_DLL herr_t H5D_chunk_cinfo_cache_reset(H5D_chunk_cached_t *last);
00598 H5_DLL herr_t H5D_chunk_create(H5D_t *dset /*in,out*/, hid_t dxpl_id);
00599 H5_DLL herr_t H5D_chunk_set_info(const H5D_t *dset);
00600 H5_DLL herr_t H5D_chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset,
00601     hid_t dapl_id);
00602 H5_DLL hbool_t H5D_chunk_is_space_alloc(const H5O_storage_t *storage);
00603 H5_DLL herr_t H5D_chunk_get_info(const H5D_t *dset, hid_t dxpl_id,
00604     const hsize_t *chunk_offset, H5D_chunk_ud_t *udata);
00605 H5_DLL void *H5D_chunk_lock(const H5D_io_info_t *io_info,
00606     H5D_chunk_ud_t *udata, hbool_t relax, unsigned *idx_hint/*in,out*/);
00607 H5_DLL herr_t H5D_chunk_unlock(const H5D_io_info_t *io_info,
00608     const H5D_chunk_ud_t *udata, hbool_t dirty, unsigned idx_hint, void *chunk,
00609     uint32_t naccessed);
00610 H5_DLL herr_t H5D_chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes);
00611 H5_DLL herr_t H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id,
00612     hbool_t full_overwrite, hsize_t old_dim[]);
00613 H5_DLL herr_t H5D_chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id,
00614     const hsize_t *old_dims);
00615 #ifdef H5_HAVE_PARALLEL
00616 H5_DLL herr_t H5D_chunk_addrmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[]);
00617 #endif /* H5_HAVE_PARALLEL */
00618 H5_DLL herr_t H5D_chunk_update_cache(H5D_t *dset, hid_t dxpl_id);
00619 H5_DLL herr_t H5D_chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src,
00620     H5O_layout_chunk_t *layout_src, H5F_t *f_dst, H5O_storage_chunk_t *storage_dst,
00621     const H5S_extent_t *ds_extent_src, const H5T_t *dt_src,
00622     const H5O_pline_t *pline_src, H5O_copy_t *cpy_info, hid_t dxpl_id);
00623 H5_DLL herr_t H5D_chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout,
00624     const H5O_pline_t *pline, hsize_t *btree_size);
00625 H5_DLL herr_t H5D_chunk_dump_index(H5D_t *dset, hid_t dxpl_id, FILE *stream);
00626 H5_DLL herr_t H5D_chunk_dest(H5F_t *f, hid_t dxpl_id, H5D_t *dset);
00627 #ifdef H5D_CHUNK_DEBUG
00628 H5_DLL herr_t H5D_chunk_stats(const H5D_t *dset, hbool_t headers);
00629 #endif /* H5D_CHUNK_DEBUG */
00630 
00631 /* Functions that operate on compact dataset storage */
00632 H5_DLL herr_t H5D_compact_fill(H5D_t *dset, hid_t dxpl_id);
00633 H5_DLL herr_t H5D_compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src,
00634     H5F_t *f_dst, H5O_storage_compact_t *storage_dst, H5T_t *src_dtype,
00635     H5O_copy_t *cpy_info, hid_t dxpl_id);
00636 
00637 /* Functions that operate on EFL (External File List)*/
00638 H5_DLL hbool_t H5D_efl_is_space_alloc(const H5O_storage_t *storage);
00639 H5_DLL herr_t H5D_efl_bh_info(H5F_t *f, hid_t dxpl_id, H5O_efl_t *efl,
00640     hsize_t *heap_size);
00641 
00642 /* Functions that perform fill value operations on datasets */
00643 H5_DLL herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf,
00644     const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id);
00645 H5_DLL herr_t H5D_fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf,
00646     hbool_t alloc_vl_during_refill,
00647     H5MM_allocate_t alloc_func, void *alloc_info,
00648     H5MM_free_t free_func, void *free_info,
00649     const H5O_fill_t *fill, const H5T_t *dset_type, hid_t dset_type_id,
00650     size_t nelmts, size_t min_buf_size, hid_t dxpl_id);
00651 H5_DLL herr_t H5D_fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts,
00652     hid_t dxpl_id);
00653 H5_DLL herr_t H5D_fill_release(H5D_fill_buf_info_t *fb_info);
00654 H5_DLL herr_t H5D_fill_term(H5D_fill_buf_info_t *fb_info);
00655 
00656 #ifdef H5_HAVE_PARALLEL
00657 
00658 #ifdef H5S_DEBUG
00659 #ifndef H5Dmpio_DEBUG
00660 #define H5Dmpio_DEBUG
00661 #endif /*H5Dmpio_DEBUG*/
00662 #endif/*H5S_DEBUG*/
00663 /* MPI-IO function to read, it will select either regular or irregular read */
00664 H5_DLL herr_t H5D_mpio_select_read(const H5D_io_info_t *io_info,
00665     const H5D_type_info_t *type_info,
00666     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00667 
00668 /* MPI-IO function to write, it will select either regular or irregular read */
00669 H5_DLL herr_t H5D_mpio_select_write(const H5D_io_info_t *io_info,
00670     const H5D_type_info_t *type_info,
00671     hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space);
00672 
00673 /* MPI-IO functions to handle contiguous collective IO */
00674 H5_DLL herr_t H5D_contig_collective_read(H5D_io_info_t *io_info,
00675     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00676     const H5S_t *mem_space, H5D_chunk_map_t *fm);
00677 H5_DLL herr_t H5D_contig_collective_write(H5D_io_info_t *io_info,
00678     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00679     const H5S_t *mem_space, H5D_chunk_map_t *fm);
00680 
00681 /* MPI-IO functions to handle chunked collective IO */
00682 H5_DLL herr_t H5D_chunk_collective_read(H5D_io_info_t *io_info,
00683     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00684     const H5S_t *mem_space, H5D_chunk_map_t *fm);
00685 H5_DLL herr_t H5D_chunk_collective_write(H5D_io_info_t *io_info,
00686     const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space,
00687     const H5S_t *mem_space, H5D_chunk_map_t *fm);
00688 
00689 /* MPI-IO function to check if a direct I/O transfer is possible between
00690  * memory and the file */
00691 H5_DLL htri_t H5D_mpio_opt_possible(const H5D_io_info_t *io_info,
00692     const H5S_t *file_space, const H5S_t *mem_space,
00693     const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm);
00694 
00695 #endif /* H5_HAVE_PARALLEL */
00696 
00697 /* Testing functions */
00698 #ifdef H5D_TESTING
00699 H5_DLL herr_t H5D_layout_version_test(hid_t did, unsigned *version);
00700 H5_DLL herr_t H5D_layout_contig_size_test(hid_t did, hsize_t *size);
00701 H5_DLL herr_t H5D_current_cache_size_test(hid_t did, size_t *nbytes_used, int *nused);
00702 #endif /* H5D_TESTING */
00703 
00704 #endif /*_H5Dpkg_H*/
00705