Merge pull request #3191 from dkjsone/bugfix

ticket #TSP-8629412 bug fix

Author: WardF

libdispatch/nchashmap.c

@@ -68,6 +68,10 @@ extern nchashkey_t hash_fast(const char*, size_t length);
 
 #define MAX(a,b) ((a) > (b) ? (a) : (b))
 
+#ifndef SIZE_MAX
+#define SIZE_MAX ((size_t)-1)
+#endif
+
 /* Forward */
 static const unsigned int NC_nprimes;
 static const unsigned int NC_primes[16386];
@@ -175,12 +179,14 @@ NC_hashmapnew(size_t startsize)
     if(startsize == 0 || startsize < MINTABLESIZE)
 	startsize = MINTABLESIZE;
     else {
+    if(startsize > SIZE_MAX / 4){nullfree(hm);return 0;}
 	startsize *= 4;
 	startsize /= 3;
 	startsize = findPrimeGreaterThan(startsize);
 	if(startsize == 0) {nullfree(hm); return 0;}
     }
     hm->table = (NC_hentry*)calloc(sizeof(NC_hentry), (size_t)startsize);
+    if(hm->table == NULL) {nullfree(hm);return 0;}
     hm->alloc = startsize;
     hm->active = 0;
     return hm;

libsrc/attr.m4

@@ -88,6 +88,9 @@ new_x_NC_attr(
 
 	assert(!(xsz == 0 && nelems != 0));
 
+	if(sz > SIZE_MAX -xsz)
+		return NULL;
+
 	sz += xsz;
 
 	attrp = (NC_attr *) malloc(sz);

libsrc/v1hpg.c

@@ -541,6 +541,8 @@ v1h_get_NC_dimarray(v1hs *gsp, NC_dimarray *ncap)
 	if(type != NC_DIMENSION)
 		return EINVAL;
 
+	if (ncap->nelems > SIZE_MAX / sizeof(NC_dim *))
+		return NC_ERANGE;
 	ncap->value = (NC_dim **) calloc(1,ncap->nelems * sizeof(NC_dim *));
 	if(ncap->value == NULL)
 		return NC_ENOMEM;
@@ -1192,13 +1194,17 @@ v1h_get_NC_vararray(v1hs *gsp, NC_vararray *ncap)
 	/* else */
 	if(type != NC_VARIABLE)
 		return EINVAL;
-
+	
+	if (ncap->nelems > SIZE_MAX / sizeof(NC_var *))
+		return NC_ERANGE;
 	ncap->value = (NC_var **) calloc(1,ncap->nelems * sizeof(NC_var *));
 	if(ncap->value == NULL)
 		return NC_ENOMEM;
 	ncap->nalloc = ncap->nelems;
 
 	ncap->hashmap = NC_hashmapnew(ncap->nelems);
+	if (ncap->hashmap == NULL)
+		return NC_ENOMEM;
 	{
 		NC_var **vpp = ncap->value;
 		NC_var *const *const end = &vpp[ncap->nelems];

libsrc/var.c

@@ -72,13 +72,21 @@ new_x_NC_var(
 	size_t ndims)
 {
 	NC_var *varp;
+
+	if (ndims > SIZE_MAX / sizeof(int))
+		return NULL;
 	const size_t o1 = M_RNDUP(ndims * sizeof(int));
+
+	if (ndims > SIZE_MAX / sizeof(size_t))
+		return NULL;
 	const size_t o2 = M_RNDUP(ndims * sizeof(size_t));
 
 #ifdef MALLOCHACK
 	const size_t sz =  M_RNDUP(sizeof(NC_var)) +
 		 o1 + o2 + ndims * sizeof(off_t);
 #else /*!MALLOCHACK*/
+	if (ndims > SIZE_MAX / sizeof(off_t))
+		return NULL;
 	const size_t o3 = ndims * sizeof(off_t);
 	const size_t sz = sizeof(NC_var);
 #endif /*!MALLOCHACK*/
@@ -477,6 +485,8 @@ NC_var_shape(NC_var *varp, const NC_dimarray *dims)
       /*if(!(shp == varp->shape && IS_RECVAR(varp)))*/
       if( shp != NULL && (shp != varp->shape || !IS_RECVAR(varp)))
 		{
+          if(product <= 0)
+            return NC_ERANGE;
           if( ((off_t)(*shp)) <= OFF_T_MAX / product )
 			{
               product *= (*shp > 0 ? (off_t)*shp : 1);
@@ -525,6 +535,8 @@ NC_check_vlen(NC_var *varp, long long vlen_max) {
     for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) {
       if(!varp->shape)
         return 0; /* Shape is undefined/NULL. */
+      if(prod <= 0)
+	    return 0; /* Multiplication operations may result in overflow */
       if ((long long)varp->shape[ii] > vlen_max / prod) {
         return 0;		/* size in bytes won't fit in a 32-bit int */
       }